Explore this codebase to find:
1. Where metrics/analytics are being pushed to Supabase
2. What tables are being used for workers, metrics, analytics
3. What fields/columns are being tracked (progress, heartbeats, language, status, etc.)
4. Any existing Supabase client setup or table schemas

Look in scripts/, workers/ folders and any Python files that interact with Supabase for metrics tracking.


I'll explore the codebase to find all Supabase-related metrics, analytics, and worker tracking. Let me start with a comprehensive search.


Bash

find /home/ubuntu/maya3_prepare -type d -name "workers" -o -type d -name "scripts"

Find workers and scripts directories


/home/ubuntu/maya3_prepare/scripts
/home/ubuntu/maya3_prepare/workers


Found 6 files
workers/seed_video_tasks.py
workers/worker.py
scripts/supabase_summary.py
scripts/normalize_languages.py
scripts/bulk_download.py
scripts/fetch_languages.py


Found 15 files
workers/worker.py
workers/worker_processing.py
scripts/legacy/clever_pipeline.py
scripts/legacy/gpu_audio_metrics_v2.py
scripts/legacy/optimized_pipeline.py
scripts/legacy/gpu_audio_metrics.py
scripts/prepare_shards.py
scripts/legacy/fast_audio_pipeline_v2.py
scripts/fast_audio_pipeline.py
tts/tts/utils/custom_logging.py
tts/tts/inference/quality_validation.py
tts/tts/training/rlhf/reward_utils.py
tts/tts/training/evaluation.py
tts/tts/training/codec/gan_training_loop.py
tts/tts/training/training_loop.py


Now let me read the key files that mention Supabase and metrics:


     1→from __future__ import annotations
     2→
     3→from dataclasses import dataclass
     4→from pathlib import Path
     5→from typing import Iterable, Optional
     6→from collections import defaultdict
     7→
     8→import psycopg
     9→
    10→
    11→ENV_PATH = Path("/root/maya3data/.env")
    12→DEFAULT_OUTPUT = Path("/root/maya3data/Summary.md")
    13→
    14→# === LANGUAGE NORMALIZATION MAP ===
    15→# Maps raw language codes/names to normalized form
    16→# Original: raw values like 'hi', 'hindi', 'ml', 'malayalam' etc.
    17→# Fixed: Normalized to consistent capitalized names for proper aggregation
    18→LANGUAGE_NORMALIZE = {
    19→    # Hindi variants
    20→    "hi": "Hindi", "hindi": "Hindi",
    21→    # Telugu variants
    22→    "te": "Telugu", "telugu": "Telugu",
    23→    # Tamil variants
    24→    "ta": "Tamil", "tamil": "Tamil",
    25→    # Malayalam variants
    26→    "ml": "Malayalam", "malayalam": "Malayalam",
    27→    # Punjabi variants
    28→    "pa": "Punjabi", "punjabi": "Punjabi",
    29→    # Kannada variants
    30→    "kn": "Kannada", "kannada": "Kannada",
    31→    # Bengali variants
    32→    "bn": "Bengali", "bengali": "Bengali",
    33→    # Gujarati variants
    34→    "gu": "Gujarati", "gujarati": "Gujarati",
    35→    # Marathi variants
    36→    "mr": "Marathi", "marathi": "Marathi",
    37→    # Odia variants
    38→    "or": "Odia", "odia": "Odia",
    39→    # Assamese
    40→    "assamese": "Assamese", "as": "Assamese",
    41→    # English variants
    42→    "en": "English", "en-us": "English", "en-gb": "English",
    43→    "en-in": "English", "english": "English",
    44→}
    45→
    46→
    47→def normalize_language(lang: Optional[str]) -> Optional[str]:
    48→    """Normalize language code/name to consistent form."""
    49→    if not lang:
    50→        return None
    51→    lang_lower = lang.strip().lower()
    52→    return LANGUAGE_NORMALIZE.get(lang_lower, lang.capitalize())
    53→
    54→
    55→def load_env_value(env_path: Path, key: str) -> Optional[str]:
    56→    for line in env_path.read_text().splitlines():
    57→        stripped = line.strip()
    58→        if not stripped or stripped.startswith("#") or "=" not in stripped:
    59→            continue
    60→        k, v = stripped.split("=", 1)
    61→        if k.strip() == key:
    62→            return v.strip()
    63→    return None
    64→
    65→
    66→def format_hours(seconds: Optional[float]) -> float:
    67→    if seconds is None:
    68→        return 0.0
    69→    return round(seconds / 3600.0, 3)
    70→
    71→
    72→@dataclass(frozen=True)
    73→class LanguageStats:
    74→    language: str
    75→    videos: int
    76→    audio_hours: float
    77→    usable_hours: float
    78→
    79→
    80→@dataclass(frozen=True)
    81→class SampleRateStats:
    82→    sample_rate_hz: Optional[int]
    83→    videos: int
    84→    audio_hours: float
    85→
    86→
    87→def fetch_language_stats(conn: psycopg.Connection) -> list[LanguageStats]:
    88→    """Fetch language stats with normalization to merge duplicates like 'hi'/'hindi'.
    89→    Only counts COMPLETED videos for accurate processed stats."""
    90→    language_expr = "nullif(btrim(language), '')"
    91→    with conn.cursor() as cur:
    92→        cur.execute(
    93→            f"""
    94→            select {language_expr} as language,
    95→                   count(*) as videos,
    96→                   sum(coalesce(audio_duration_sec, source_duration_min * 60.0)) as audio_seconds,
    97→                   sum(coalesce(usable_duration_sec, 0)) as usable_seconds
    98→            from videos
    99→            where {language_expr} is not null
   100→              and status = 'COMPLETED'
   101→            group by {language_expr};
   102→            """
   103→        )
   104→        rows = cur.fetchall()
   105→
   106→    # Aggregate by normalized language name
   107→    aggregated: dict[str, dict] = defaultdict(lambda: {"videos": 0, "audio_sec": 0.0, "usable_sec": 0.0})
   108→    for row in rows:
   109→        raw_lang = row[0]
   110→        norm_lang = normalize_language(raw_lang)
   111→        if norm_lang:
   112→            aggregated[norm_lang]["videos"] += int(row[1])
   113→            aggregated[norm_lang]["audio_sec"] += float(row[2] or 0)
   114→            aggregated[norm_lang]["usable_sec"] += float(row[3] or 0)
   115→
   116→    result = [
   117→        LanguageStats(
   118→            language=lang,
   119→            videos=data["videos"],
   120→            audio_hours=format_hours(data["audio_sec"]),
   121→            usable_hours=format_hours(data["usable_sec"]),
   122→        )
   123→        for lang, data in aggregated.items()
   124→    ]
   125→    return sorted(result, key=lambda x: x.videos, reverse=True)
   126→
   127→
   128→def fetch_sample_rate_stats(conn: psycopg.Connection) -> list[SampleRateStats]:
   129→    """Only counts COMPLETED videos for accurate sample rate stats."""
   130→    with conn.cursor() as cur:
   131→        cur.execute(
   132→            """
   133→            select audio_native_sample_rate,
   134→                   count(*) as videos,
   135→                   sum(coalesce(audio_duration_sec, source_duration_min * 60.0)) as audio_seconds
   136→            from videos
   137→            where status = 'COMPLETED'
   138→            group by audio_native_sample_rate
   139→            order by audio_native_sample_rate nulls last;
   140→            """
   141→        )
   142→        rows = cur.fetchall()
   143→
   144→    return [
   145→        SampleRateStats(
   146→            sample_rate_hz=row[0],
   147→            videos=int(row[1]),
   148→            audio_hours=format_hours(row[2]),
   149→        )
   150→        for row in rows
   151→    ]
   152→
   153→
   154→def fetch_totals(conn: psycopg.Connection) -> dict[str, int]:
   155→    """Fetch totals - completed videos only for main stats."""
   156→    language_expr = "nullif(btrim(language), '')"
   157→    with conn.cursor() as cur:
   158→        cur.execute("select count(*) from videos where status = 'COMPLETED';")
   159→        total_videos = int(cur.fetchone()[0])
   160→
   161→        # Get unique raw languages and normalize to count unique (COMPLETED only)
   162→        cur.execute(
   163→            f"""
   164→            select distinct {language_expr}
   165→            from videos
   166→            where {language_expr} is not null
   167→              and status = 'COMPLETED';
   168→            """
   169→        )
   170→        raw_languages = [row[0] for row in cur.fetchall()]
   171→        normalized_languages = set(normalize_language(lang) for lang in raw_languages if normalize_language(lang))
   172→        total_languages = len(normalized_languages)
   173→        total_raw_languages = len(raw_languages)
   174→
   175→        cur.execute(
   176→            f"""
   177→            select count(*)
   178→            from videos
   179→            where {language_expr} is null
   180→              and status = 'COMPLETED';
   181→            """
   182→        )
   183→        missing_language = int(cur.fetchone()[0])
   184→
   185→    return {
   186→        "total_videos": total_videos,
   187→        "total_languages": total_languages,
   188→        "total_raw_languages": total_raw_languages,
   189→        "missing_language": missing_language,
   190→    }
   191→
   192→
   193→def fetch_status_breakdown(conn: psycopg.Connection) -> list[tuple[str, int]]:
   194→    """Breakdown of videos by processing status."""
   195→    with conn.cursor() as cur:
   196→        cur.execute(
   197→            """
   198→            select coalesce(status, 'NULL') as status, count(*) as cnt
   199→            from videos
   200→            group by status
   201→            order by cnt desc;
   202→            """
   203→        )
   204→        return [(row[0], int(row[1])) for row in cur.fetchall()]
   205→
   206→
   207→def fetch_classification_breakdown(conn: psycopg.Connection) -> list[tuple[str, int, float]]:
   208→    """Breakdown by classification (podcast, interview, etc). COMPLETED only."""
   209→    with conn.cursor() as cur:
   210→        cur.execute(
   211→            """
   212→            select coalesce(classification, 'unknown') as classification,
   213→                   count(*) as cnt,
   214→                   sum(coalesce(audio_duration_sec, source_duration_min * 60.0)) as audio_seconds
   215→            from videos
   216→            where status = 'COMPLETED'
   217→            group by classification
   218→            order by cnt desc;
   219→            """
   220→        )
   221→        return [(row[0], int(row[1]), format_hours(row[2])) for row in cur.fetchall()]
   222→
   223→
   224→def fetch_audio_totals(conn: psycopg.Connection) -> dict[str, float]:
   225→    """Only counts COMPLETED videos for accurate audio totals."""
   226→    with conn.cursor() as cur:
   227→        cur.execute(
   228→            """
   229→            select sum(coalesce(audio_duration_sec, source_duration_min * 60.0)) as audio_seconds,
   230→                   sum(coalesce(usable_duration_sec, 0)) as usable_seconds
   231→            from videos
   232→            where status = 'COMPLETED';
   233→            """
   234→        )
   235→        audio_seconds, usable_seconds = cur.fetchone()
   236→
   237→    return {
   238→        "audio_hours": format_hours(audio_seconds),
   239→        "usable_hours": format_hours(usable_seconds),
   240→    }
   241→
   242→
   243→def render_markdown(
   244→    totals: dict[str, int],
   245→    audio_totals: dict[str, float],
   246→    language_stats: Iterable[LanguageStats],
   247→    sample_rate_stats: Iterable[SampleRateStats],
   248→    status_breakdown: list[tuple[str, int]],
   249→    classification_breakdown: list[tuple[str, int, float]],
   250→) -> str:
   251→    lines: list[str] = []
   252→    lines.append("# Supabase Videos Summary")
   253→    lines.append("")
   254→    lines.append("## Totals")
   255→    lines.append(f"- **Total Videos**: {totals['total_videos']:,}")
   256→    lines.append(f"- **Unique Languages** (normalized): {totals['total_languages']}")
   257→    lines.append(f"- **Raw Language Variants**: {totals['total_raw_languages']}")
   258→    lines.append(f"- **Missing Language**: {totals['missing_language']:,}")
   259→    lines.append(f"- **Total Audio Hours**: {audio_totals['audio_hours']:,.2f}")
   260→    lines.append(f"- **Usable Audio Hours**: {audio_totals['usable_hours']:,.2f}")
   261→    lines.append("")
   262→
   263→    lines.append("## Status Breakdown")
   264→    lines.append("| Status | Videos | Percentage |")
   265→    lines.append("| --- | ---: | ---: |")
   266→    total_vids = totals['total_videos']
   267→    for status, count in status_breakdown:
   268→        pct = (count / total_vids * 100) if total_vids > 0 else 0
   269→        lines.append(f"| {status} | {count:,} | {pct:.1f}% |")
   270→    lines.append("")
   271→
   272→    lines.append("## Classification Breakdown")
   273→    lines.append("| Classification | Videos | Audio Hours |")
   274→    lines.append("| --- | ---: | ---: |")
   275→    for classification, count, hours in classification_breakdown:
   276→        lines.append(f"| {classification} | {count:,} | {hours:,.2f} |")
   277→    lines.append("")
   278→
   279→    lines.append("## Language Breakdown (Normalized)")
   280→    lines.append("| Language | Videos | Audio Hours | Usable Hours |")
   281→    lines.append("| --- | ---: | ---: | ---: |")
   282→    for row in language_stats:
   283→        lines.append(
   284→            f"| {row.language} | {row.videos:,} | {row.audio_hours:,.2f} | {row.usable_hours:,.2f} |"
   285→        )
   286→    lines.append("")
   287→
   288→    lines.append("## Native Sample Rate Breakdown")
   289→    lines.append("| Sample Rate (Hz) | Videos | Audio Hours |")
   290→    lines.append("| --- | ---: | ---: |")
   291→    for row in sample_rate_stats:
   292→        sample_rate = "unknown" if row.sample_rate_hz is None else f"{row.sample_rate_hz:,}"
   293→        lines.append(f"| {sample_rate} | {row.videos:,} | {row.audio_hours:,.2f} |")
   294→    lines.append("")
   295→    return "\n".join(lines)
   296→
   297→
   298→def main(output_path: Optional[Path] = None) -> None:
   299→    db_url = load_env_value(ENV_PATH, "DATABASE_URL")
   300→    if not db_url:
   301→        raise SystemExit("DATABASE_URL not found in .env")
   302→
   303→    print("Connecting to Supabase...")
   304→    with psycopg.connect(db_url) as conn:
   305→        print("Fetching totals...")
   306→        totals = fetch_totals(conn)
   307→        print("Fetching audio totals...")
   308→        audio_totals = fetch_audio_totals(conn)
   309→        print("Fetching language stats (with normalization)...")
   310→        language_stats = fetch_language_stats(conn)
   311→        print("Fetching sample rate stats...")
   312→        sample_rate_stats = fetch_sample_rate_stats(conn)
   313→        print("Fetching status breakdown...")
   314→        status_breakdown = fetch_status_breakdown(conn)
   315→        print("Fetching classification breakdown...")
   316→        classification_breakdown = fetch_classification_breakdown(conn)
   317→
   318→    markdown = render_markdown(
   319→        totals, audio_totals, language_stats, sample_rate_stats,
   320→        status_breakdown, classification_breakdown
   321→    )
   322→
   323→    if output_path:
   324→        output_path.write_text(markdown)
   325→        print(f"Summary written to {output_path}")
   326→    else:
   327→        print("\n" + "="*60)
   328→        print(markdown)
   329→
   330→
   331→if __name__ == "__main__":
   332→    import sys
   333→    out = Path(sys.argv[1]) if len(sys.argv) > 1 else None
   334→    main(out)
   335→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, Optional
from collections import defaultdict

import psycopg

ENV_PATH = Path("/root/maya3data/.env")
DEFAULT_OUTPUT = Path("/root/maya3data/Summary.md")

# === LANGUAGE NORMALIZATION MAP ===
# Maps raw language codes/names to normalized form
# Original: raw values like 'hi', 'hindi', 'ml', 'malayalam' etc.
# Fixed: Normalized to consistent capitalized names for proper aggregation
LANGUAGE_NORMALIZE = {
    # Hindi variants
    "hi": "Hindi", "hindi": "Hindi",
    # Telugu variants
    "te": "Telugu", "telugu": "Telugu",
    # Tamil variants
    "ta": "Tamil", "tamil": "Tamil",
    # Malayalam variants
    "ml": "Malayalam", "malayalam": "Malayalam",
    # Punjabi variants
    "pa": "Punjabi", "punjabi": "Punjabi",
    # Kannada variants
    "kn": "Kannada", "kannada": "Kannada",
    # Bengali variants
    "bn": "Bengali", "bengali": "Bengali",
    # Gujarati variants
    "gu": "Gujarati", "gujarati": "Gujarati",
    # Marathi variants
    "mr": "Marathi", "marathi": "Marathi",
    # Odia variants
    "or": "Odia", "odia": "Odia",
    # Assamese
    "assamese": "Assamese", "as": "Assamese",
    # English variants
    "en": "English", "en-us": "English", "en-gb": "English",
    "en-in": "English", "english": "English",
}

def normalize_language(lang: Optional[str]) -> Optional[str]:
    """Normalize language code/name to consistent form."""
    if not lang:
        return None
    lang_lower = lang.strip().lower()
    return LANGUAGE_NORMALIZE.get(lang_lower, lang.capitalize())

def load_env_value(env_path: Path, key: str) -> Optional[str]:
    for line in env_path.read_text().splitlines():
        stripped = line.strip()
        if not stripped or stripped.startswith("#") or "=" not in stripped:
            continue
        k, v = stripped.split("=", 1)
        if k.strip() == key:
            return v.strip()
    return None

def format_hours(seconds: Optional[float]) -> float:
    if seconds is None:
        return 0.0
    return round(seconds / 3600.0, 3)

@dataclass(frozen=True)
class LanguageStats:
    language: str
    videos: int
    audio_hours: float
    usable_hours: float

@dataclass(frozen=True)
class SampleRateStats:
    sample_rate_hz: Optional[int]
    videos: int
    audio_hours: float

def fetch_language_stats(conn: psycopg.Connection) -> list[LanguageStats]:
    """Fetch language stats with normalization to merge duplicates like 'hi'/'hindi'.
    Only counts COMPLETED videos for accurate processed stats."""
    language_expr = "nullif(btrim(language), '')"
    with conn.cursor() as cur:
        cur.execute(
            f"""
            select {language_expr} as language,
                   count(*) as videos,
                   sum(coalesce(audio_duration_sec, source_duration_min * 60.0)) as audio_seconds,
                   sum(coalesce(usable_duration_sec, 0)) as usable_seconds
            from videos
            where {language_expr} is not null
              and status = 'COMPLETED'
            group by {language_expr};
            """
        )
        rows = cur.fetchall()

    # Aggregate by normalized language name
    aggregated: dict[str, dict] = defaultdict(lambda: {"videos": 0, "audio_sec": 0.0, "usable_sec": 0.0})
    for row in rows:
        raw_lang = row[0]
        norm_lang = normalize_language(raw_lang)
        if norm_lang:
            aggregated[norm_lang]["videos"] += int(row[1])
            aggregated[norm_lang]["audio_sec"] += float(row[2] or 0)
            aggregated[norm_lang]["usable_sec"] += float(row[3] or 0)

    result = [
        LanguageStats(
            language=lang,
            videos=data["videos"],
            audio_hours=format_hours(data["audio_sec"]),
            usable_hours=format_hours(data["usable_sec"]),
        )
        for lang, data in aggregated.items()
    ]
    return sorted(result, key=lambda x: x.videos, reverse=True)

def fetch_sample_rate_stats(conn: psycopg.Connection) -> list[SampleRateStats]:
    """Only counts COMPLETED videos for accurate sample rate stats."""
    with conn.cursor() as cur:
        cur.execute(
            """
            select audio_native_sample_rate,
                   count(*) as videos,
                   sum(coalesce(audio_duration_sec, source_duration_min * 60.0)) as audio_seconds
            from videos
            where status = 'COMPLETED'
            group by audio_native_sample_rate
            order by audio_native_sample_rate nulls last;
            """
        )
        rows = cur.fetchall()

    return [
        SampleRateStats(
            sample_rate_hz=row[0],
            videos=int(row[1]),
            audio_hours=format_hours(row[2]),
        )
        for row in rows
    ]

def fetch_totals(conn: psycopg.Connection) -> dict[str, int]:
    """Fetch totals - completed videos only for main stats."""
    language_expr = "nullif(btrim(language), '')"
    with conn.cursor() as cur:
        cur.execute("select count(*) from videos where status = 'COMPLETED';")
        total_videos = int(cur.fetchone()[0])

        # Get unique raw languages and normalize to count unique (COMPLETED only)
        cur.execute(
            f"""
            select distinct {language_expr}
            from videos
            where {language_expr} is not null
              and status = 'COMPLETED';
            """
        )
        raw_languages = [row[0] for row in cur.fetchall()]
        normalized_languages = set(normalize_language(lang) for lang in raw_languages if normalize_language(lang))
        total_languages = len(normalized_languages)
        total_raw_languages = len(raw_languages)

        cur.execute(
            f"""
            select count(*)
            from videos
            where {language_expr} is null
              and status = 'COMPLETED';
            """
        )
        missing_language = int(cur.fetchone()[0])

    return {
        "total_videos": total_videos,
        "total_languages": total_languages,
        "total_raw_languages": total_raw_languages,
        "missing_language": missing_language,
    }

def fetch_status_breakdown(conn: psycopg.Connection) -> list[tuple[str, int]]:
    """Breakdown of videos by processing status."""
    with conn.cursor() as cur:
        cur.execute(
            """
            select coalesce(status, 'NULL') as status, count(*) as cnt
            from videos
            group by status
            order by cnt desc;
            """
        )
        return [(row[0], int(row[1])) for row in cur.fetchall()]

def fetch_classification_breakdown(conn: psycopg.Connection) -> list[tuple[str, int, float]]:
    """Breakdown by classification (podcast, interview, etc). COMPLETED only."""
    with conn.cursor() as cur:
        cur.execute(
            """
            select coalesce(classification, 'unknown') as classification,
                   count(*) as cnt,
                   sum(coalesce(audio_duration_sec, source_duration_min * 60.0)) as audio_seconds
            from videos
            where status = 'COMPLETED'
            group by classification
            order by cnt desc;
            """
        )
        return [(row[0], int(row[1]), format_hours(row[2])) for row in cur.fetchall()]

def fetch_audio_totals(conn: psycopg.Connection) -> dict[str, float]:
    """Only counts COMPLETED videos for accurate audio totals."""
    with conn.cursor() as cur:
        cur.execute(
            """
            select sum(coalesce(audio_duration_sec, source_duration_min * 60.0)) as audio_seconds,
                   sum(coalesce(usable_duration_sec, 0)) as usable_seconds
            from videos
            where status = 'COMPLETED';
            """
        )
        audio_seconds, usable_seconds = cur.fetchone()

    return {
        "audio_hours": format_hours(audio_seconds),
        "usable_hours": format_hours(usable_seconds),
    }

def render_markdown(
    totals: dict[str, int],
    audio_totals: dict[str, float],
    language_stats: Iterable[LanguageStats],
    sample_rate_stats: Iterable[SampleRateStats],
    status_breakdown: list[tuple[str, int]],
    classification_breakdown: list[tuple[str, int, float]],
) -> str:
    lines: list[str] = []
    lines.append("# Supabase Videos Summary")
    lines.append("")
    lines.append("## Totals")
    lines.append(f"- **Total Videos**: {totals['total_videos']:,}")
    lines.append(f"- **Unique Languages** (normalized): {totals['total_languages']}")
    lines.append(f"- **Raw Language Variants**: {totals['total_raw_languages']}")
    lines.append(f"- **Missing Language**: {totals['missing_language']:,}")
    lines.append(f"- **Total Audio Hours**: {audio_totals['audio_hours']:,.2f}")
    lines.append(f"- **Usable Audio Hours**: {audio_totals['usable_hours']:,.2f}")
    lines.append("")

    lines.append("## Status Breakdown")
    lines.append("| Status | Videos | Percentage |")
    lines.append("| --- | ---: | ---: |")
    total_vids = totals['total_videos']
    for status, count in status_breakdown:
        pct = (count / total_vids * 100) if total_vids > 0 else 0
        lines.append(f"| {status} | {count:,} | {pct:.1f}% |")
    lines.append("")

    lines.append("## Classification Breakdown")
    lines.append("| Classification | Videos | Audio Hours |")
    lines.append("| --- | ---: | ---: |")
    for classification, count, hours in classification_breakdown:
        lines.append(f"| {classification} | {count:,} | {hours:,.2f} |")
    lines.append("")

    lines.append("## Language Breakdown (Normalized)")
    lines.append("| Language | Videos | Audio Hours | Usable Hours |")
    lines.append("| --- | ---: | ---: | ---: |")
    for row in language_stats:
        lines.append(
            f"| {row.language} | {row.videos:,} | {row.audio_hours:,.2f} | {row.usable_hours:,.2f} |"
        )
    lines.append("")

    lines.append("## Native Sample Rate Breakdown")
    lines.append("| Sample Rate (Hz) | Videos | Audio Hours |")
    lines.append("| --- | ---: | ---: |")
    for row in sample_rate_stats:
        sample_rate = "unknown" if row.sample_rate_hz is None else f"{row.sample_rate_hz:,}"
        lines.append(f"| {sample_rate} | {row.videos:,} | {row.audio_hours:,.2f} |")
    lines.append("")
    return "\n".join(lines)

def main(output_path: Optional[Path] = None) -> None:
    db_url = load_env_value(ENV_PATH, "DATABASE_URL")
    if not db_url:
        raise SystemExit("DATABASE_URL not found in .env")

    print("Connecting to Supabase...")
    with psycopg.connect(db_url) as conn:
        print("Fetching totals...")
        totals = fetch_totals(conn)
        print("Fetching audio totals...")
        audio_totals = fetch_audio_totals(conn)
        print("Fetching language stats (with normalization)...")
        language_stats = fetch_language_stats(conn)
        print("Fetching sample rate stats...")
        sample_rate_stats = fetch_sample_rate_stats(conn)
        print("Fetching status breakdown...")
        status_breakdown = fetch_status_breakdown(conn)
        print("Fetching classification breakdown...")
        classification_breakdown = fetch_classification_breakdown(conn)

    markdown = render_markdown(
        totals, audio_totals, language_stats, sample_rate_stats,
        status_breakdown, classification_breakdown
    )

    if output_path:
        output_path.write_text(markdown)
        print(f"Summary written to {output_path}")
    else:
        print("\n" + "="*60)
        print(markdown)

if __name__ == "__main__":
    import sys
    out = Path(sys.argv[1]) if len(sys.argv) > 1 else None
    main(out)


     1→"""Configuration management for workers."""
     2→
     3→from __future__ import annotations
     4→
     5→import os
     6→from dataclasses import dataclass, field
     7→from pathlib import Path
     8→from typing import Optional
     9→
    10→
    11→@dataclass
    12→class WorkerConfig:
    13→    """Worker configuration with defaults and environment overrides."""
    14→
    15→    # Worker identity
    16→    worker_id: str = ""
    17→    language: str = ""
    18→    auto_language: bool = False
    19→
    20→    # Processing settings
    21→    target_videos: int = 100  # Videos per shard (approx 5GB)
    22→    lease_minutes: int = 60  # Lease duration for claims
    23→    heartbeat_interval_sec: int = 15  # Heartbeat frequency
    24→
    25→    # Directory settings
    26→    work_dir: Path = field(default_factory=lambda: Path("/tmp/worker"))
    27→    output_dir: Path = field(default_factory=lambda: Path("/tmp/worker_output"))
    28→
    29→    # Storage management
    30→    max_disk_usage_gb: float = 150.0  # Max disk usage before cleanup
    31→    cleanup_threshold_gb: float = 100.0  # Target after cleanup
    32→    cleanup_check_interval_sec: int = 60  # Storage check frequency
    33→
    34→    # Parallelism
    35→    download_workers: int = 8
    36→    segment_workers: int = 8
    37→    upload_workers: int = 4
    38→
    39→    # Output settings
    40→    output_format: str = "flac"  # flac or wav
    41→    target_shard_size_mb: int = 0  # 0 disables size-based splitting
    42→
    43→    # Processing options
    44→    chunk_enabled: bool = True
    45→    chunk_min_sec: float = 6.0
    46→    chunk_max_sec: float = 10.0
    47→    chunk_threshold_sec: float = 12.0
    48→    chunk_all_buckets: bool = False
    49→
    50→    silence_enabled: bool = True
    51→    silence_frame_ms: float = 50.0
    52→    silence_hop_ms: float = 10.0
    53→    silence_search_sec: float = 0.5
    54→    silence_percentile: float = 20.0
    55→    silence_db_drop: float = 20.0
    56→
    57→    val_ratio: float = 0.01
    58→    seed: int = 17
    59→
    60→    # Retry settings
    61→    max_retries: int = 3
    62→    retry_delay_sec: float = 5.0
    63→
    64→    # Run mode
    65→    once: bool = False  # Process single shard and exit
    66→    dry_run: bool = False  # Don't actually process
    67→
    68→    @classmethod
    69→    def from_env(cls) -> "WorkerConfig":
    70→        """Create config from environment variables."""
    71→        return cls(
    72→            worker_id=os.environ.get("WORKER_ID", ""),
    73→            language=os.environ.get("LANGUAGE", ""),
    74→            auto_language=os.environ.get("AUTO_LANGUAGE", "").lower()
    75→            in ("1", "true", "yes"),
    76→            target_videos=int(os.environ.get("TARGET_VIDEOS", "100")),
    77→            lease_minutes=int(os.environ.get("LEASE_MINUTES", "60")),
    78→            heartbeat_interval_sec=int(os.environ.get("HEARTBEAT_INTERVAL_SEC", "15")),
    79→            work_dir=Path(os.environ.get("WORK_DIR", "/tmp/worker")),
    80→            output_dir=Path(os.environ.get("OUTPUT_DIR", "/tmp/worker_output")),
    81→            max_disk_usage_gb=float(os.environ.get("MAX_DISK_USAGE_GB", "150")),
    82→            cleanup_threshold_gb=float(os.environ.get("CLEANUP_THRESHOLD_GB", "100")),
    83→            cleanup_check_interval_sec=int(os.environ.get("CLEANUP_CHECK_INTERVAL_SEC", "60")),
    84→            download_workers=int(os.environ.get("DOWNLOAD_WORKERS", "8")),
    85→            segment_workers=int(os.environ.get("SEGMENT_WORKERS", "8")),
    86→            upload_workers=int(os.environ.get("UPLOAD_WORKERS", "4")),
    87→            output_format=os.environ.get("OUTPUT_FORMAT", "flac"),
    88→            target_shard_size_mb=int(
    89→                os.environ.get("TARGET_SHARD_SIZE_MB", os.environ.get("SHARD_SIZE_MB", "0"))
    90→            ),
    91→            max_retries=int(os.environ.get("MAX_RETRIES", "3")),
    92→            retry_delay_sec=float(os.environ.get("RETRY_DELAY_SEC", "5")),
    93→            once=os.environ.get("ONCE", "").lower() in ("1", "true", "yes"),
    94→            dry_run=os.environ.get("DRY_RUN", "").lower() in ("1", "true", "yes"),
    95→        )
    96→
    97→
    98→@dataclass
    99→class EnvConfig:
   100→    """Environment configuration from .env file."""
   101→
   102→    # Database
   103→    database_url: str = ""
   104→
   105→    # R2 Storage
   106→    r2_endpoint_url: str = ""
   107→    r2_access_key_id: str = ""
   108→    r2_secret_access_key: str = ""
   109→    r2_cleaned_bucket: str = ""  # Source bucket with video tars
   110→    r2_bucket_shards: str = ""  # Destination bucket for shards
   111→    r2_prefix: Optional[str] = None
   112→
   113→    @classmethod
   114→    def from_file(cls, env_path: Path) -> "EnvConfig":
   115→        """Load from .env file."""
   116→        env: dict[str, str] = {}
   117→        if env_path.exists():
   118→            for line in env_path.read_text().splitlines():
   119→                stripped = line.strip()
   120→                if not stripped or stripped.startswith("#") or "=" not in stripped:
   121→                    continue
   122→                if stripped.startswith("export "):
   123→                    stripped = stripped[len("export "):]
   124→                key, value = stripped.split("=", 1)
   125→                env[key.strip()] = value.strip().strip('"').strip("'")
   126→
   127→        return cls(
   128→            database_url=env.get("DATABASE_URL", ""),
   129→            r2_endpoint_url=env.get("R2_ENDPOINT_URL", ""),
   130→            r2_access_key_id=env.get("R2_ACCESS_KEY_ID", ""),
   131→            r2_secret_access_key=env.get("R2_SECRET_ACCESS_KEY", ""),
   132→            r2_cleaned_bucket=env.get("R2_CLEANED_BUCKET") or env.get("R2_ClEANED_BUCKET", ""),
   133→            r2_bucket_shards=env.get("R2_BUCKET_SHARDS")
   134→            or env.get("R2_PROCESSED_BUCKET", ""),
   135→            r2_prefix=env.get("R2_PREFIX"),
   136→        )
   137→
   138→    @classmethod
   139→    def from_env_vars(cls) -> "EnvConfig":
   140→        """Load from environment variables."""
   141→        return cls(
   142→            database_url=os.environ.get("DATABASE_URL", ""),
   143→            r2_endpoint_url=os.environ.get("R2_ENDPOINT_URL", ""),
   144→            r2_access_key_id=os.environ.get("R2_ACCESS_KEY_ID", ""),
   145→            r2_secret_access_key=os.environ.get("R2_SECRET_ACCESS_KEY", ""),
   146→            r2_cleaned_bucket=os.environ.get("R2_CLEANED_BUCKET") or os.environ.get("R2_ClEANED_BUCKET", ""),
   147→            r2_bucket_shards=os.environ.get("R2_BUCKET_SHARDS")
   148→            or os.environ.get("R2_PROCESSED_BUCKET", ""),
   149→            r2_prefix=os.environ.get("R2_PREFIX"),
   150→        )
   151→
   152→    def validate(self) -> list[str]:
   153→        """Validate configuration, return list of errors."""
   154→        errors = []
   155→        if not self.database_url:
   156→            errors.append("DATABASE_URL is required")
   157→        if not self.r2_endpoint_url:
   158→            errors.append("R2_ENDPOINT_URL is required")
   159→        if not self.r2_access_key_id:
   160→            errors.append("R2_ACCESS_KEY_ID is required")
   161→        if not self.r2_secret_access_key:
   162→            errors.append("R2_SECRET_ACCESS_KEY is required")
   163→        if not self.r2_cleaned_bucket:
   164→            errors.append("R2_CLEANED_BUCKET is required")
   165→        if not self.r2_bucket_shards:
   166→            errors.append("R2_BUCKET_SHARDS is required")
   167→        return errors
   168→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

"""Configuration management for workers."""

from __future__ import annotations

import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional

@dataclass
class WorkerConfig:
    """Worker configuration with defaults and environment overrides."""

    # Worker identity
    worker_id: str = ""
    language: str = ""
    auto_language: bool = False

    # Processing settings
    target_videos: int = 100  # Videos per shard (approx 5GB)
    lease_minutes: int = 60  # Lease duration for claims
    heartbeat_interval_sec: int = 15  # Heartbeat frequency

    # Directory settings
    work_dir: Path = field(default_factory=lambda: Path("/tmp/worker"))
    output_dir: Path = field(default_factory=lambda: Path("/tmp/worker_output"))

    # Storage management
    max_disk_usage_gb: float = 150.0  # Max disk usage before cleanup
    cleanup_threshold_gb: float = 100.0  # Target after cleanup
    cleanup_check_interval_sec: int = 60  # Storage check frequency

    # Parallelism
    download_workers: int = 8
    segment_workers: int = 8
    upload_workers: int = 4

    # Output settings
    output_format: str = "flac"  # flac or wav
    target_shard_size_mb: int = 0  # 0 disables size-based splitting

    # Processing options
    chunk_enabled: bool = True
    chunk_min_sec: float = 6.0
    chunk_max_sec: float = 10.0
    chunk_threshold_sec: float = 12.0
    chunk_all_buckets: bool = False

    silence_enabled: bool = True
    silence_frame_ms: float = 50.0
    silence_hop_ms: float = 10.0
    silence_search_sec: float = 0.5
    silence_percentile: float = 20.0
    silence_db_drop: float = 20.0

    val_ratio: float = 0.01
    seed: int = 17

    # Retry settings
    max_retries: int = 3
    retry_delay_sec: float = 5.0

    # Run mode
    once: bool = False  # Process single shard and exit
    dry_run: bool = False  # Don't actually process

    @classmethod
    def from_env(cls) -> "WorkerConfig":
        """Create config from environment variables."""
        return cls(
            worker_id=os.environ.get("WORKER_ID", ""),
            language=os.environ.get("LANGUAGE", ""),
            auto_language=os.environ.get("AUTO_LANGUAGE", "").lower()
            in ("1", "true", "yes"),
            target_videos=int(os.environ.get("TARGET_VIDEOS", "100")),
            lease_minutes=int(os.environ.get("LEASE_MINUTES", "60")),
            heartbeat_interval_sec=int(os.environ.get("HEARTBEAT_INTERVAL_SEC", "15")),
            work_dir=Path(os.environ.get("WORK_DIR", "/tmp/worker")),
            output_dir=Path(os.environ.get("OUTPUT_DIR", "/tmp/worker_output")),
            max_disk_usage_gb=float(os.environ.get("MAX_DISK_USAGE_GB", "150")),
            cleanup_threshold_gb=float(os.environ.get("CLEANUP_THRESHOLD_GB", "100")),
            cleanup_check_interval_sec=int(os.environ.get("CLEANUP_CHECK_INTERVAL_SEC", "60")),
            download_workers=int(os.environ.get("DOWNLOAD_WORKERS", "8")),
            segment_workers=int(os.environ.get("SEGMENT_WORKERS", "8")),
            upload_workers=int(os.environ.get("UPLOAD_WORKERS", "4")),
            output_format=os.environ.get("OUTPUT_FORMAT", "flac"),
            target_shard_size_mb=int(
                os.environ.get("TARGET_SHARD_SIZE_MB", os.environ.get("SHARD_SIZE_MB", "0"))
            ),
            max_retries=int(os.environ.get("MAX_RETRIES", "3")),
            retry_delay_sec=float(os.environ.get("RETRY_DELAY_SEC", "5")),
            once=os.environ.get("ONCE", "").lower() in ("1", "true", "yes"),
            dry_run=os.environ.get("DRY_RUN", "").lower() in ("1", "true", "yes"),
        )

@dataclass
class EnvConfig:
    """Environment configuration from .env file."""

    # Database
    database_url: str = ""

    # R2 Storage
    r2_endpoint_url: str = ""
    r2_access_key_id: str = ""
    r2_secret_access_key: str = ""
    r2_cleaned_bucket: str = ""  # Source bucket with video tars
    r2_bucket_shards: str = ""  # Destination bucket for shards
    r2_prefix: Optional[str] = None

    @classmethod
    def from_file(cls, env_path: Path) -> "EnvConfig":
        """Load from .env file."""
        env: dict[str, str] = {}
        if env_path.exists():
            for line in env_path.read_text().splitlines():
                stripped = line.strip()
                if not stripped or stripped.startswith("#") or "=" not in stripped:
                    continue
                if stripped.startswith("export "):
                    stripped = stripped[len("export "):]
                key, value = stripped.split("=", 1)
                env[key.strip()] = value.strip().strip('"').strip("'")

        return cls(
            database_url=env.get("DATABASE_URL", ""),
            r2_endpoint_url=env.get("R2_ENDPOINT_URL", ""),
            r2_access_key_id=env.get("R2_ACCESS_KEY_ID", ""),
            r2_secret_access_key=env.get("R2_SECRET_ACCESS_KEY", ""),
            r2_cleaned_bucket=env.get("R2_CLEANED_BUCKET") or env.get("R2_ClEANED_BUCKET", ""),
            r2_bucket_shards=env.get("R2_BUCKET_SHARDS")
            or env.get("R2_PROCESSED_BUCKET", ""),
            r2_prefix=env.get("R2_PREFIX"),
        )

    @classmethod
    def from_env_vars(cls) -> "EnvConfig":
        """Load from environment variables."""
        return cls(
            database_url=os.environ.get("DATABASE_URL", ""),
            r2_endpoint_url=os.environ.get("R2_ENDPOINT_URL", ""),
            r2_access_key_id=os.environ.get("R2_ACCESS_KEY_ID", ""),
            r2_secret_access_key=os.environ.get("R2_SECRET_ACCESS_KEY", ""),
            r2_cleaned_bucket=os.environ.get("R2_CLEANED_BUCKET") or os.environ.get("R2_ClEANED_BUCKET", ""),
            r2_bucket_shards=os.environ.get("R2_BUCKET_SHARDS")
            or os.environ.get("R2_PROCESSED_BUCKET", ""),
            r2_prefix=os.environ.get("R2_PREFIX"),
        )

    def validate(self) -> list[str]:
        """Validate configuration, return list of errors."""
        errors = []
        if not self.database_url:
            errors.append("DATABASE_URL is required")
        if not self.r2_endpoint_url:
            errors.append("R2_ENDPOINT_URL is required")
        if not self.r2_access_key_id:
            errors.append("R2_ACCESS_KEY_ID is required")
        if not self.r2_secret_access_key:
            errors.append("R2_SECRET_ACCESS_KEY is required")
        if not self.r2_cleaned_bucket:
            errors.append("R2_CLEANED_BUCKET is required")
        if not self.r2_bucket_shards:
            errors.append("R2_BUCKET_SHARDS is required")
        return errors