     1→#!/usr/bin/env python3
     2→"""Seed video_tasks from the existing videos table (Supabase/Postgres)."""
     3→
     4→from __future__ import annotations
     5→
     6→import argparse
     7→from pathlib import Path
     8→
     9→import psycopg
    10→import sys
    11→
    12→ROOT = Path(__file__).resolve().parents[1]
    13→sys.path.insert(0, str(ROOT / "scripts"))
    14→
    15→from prepare_shards import load_env
    16→
    17→
    18→def parse_args() -> argparse.Namespace:
    19→    parser = argparse.ArgumentParser(description="Seed video_tasks table.")
    20→    parser.add_argument("--language")
    21→    parser.add_argument(
    22→        "--all-languages",
    23→        action="store_true",
    24→        help="Seed tasks for all languages (ignores --language)",
    25→    )
    26→    parser.add_argument("--limit", type=int, default=None)
    27→    parser.add_argument("--dry-run", action="store_true")
    28→    return parser.parse_args()
    29→
    30→
    31→def main() -> None:
    32→    args = parse_args()
    33→    env = load_env(Path(".env"))
    34→    db_url = env.get("DATABASE_URL")
    35→    if not db_url:
    36→        raise SystemExit("DATABASE_URL missing in .env")
    37→
    38→    with psycopg.connect(db_url) as conn:
    39→        with conn.cursor() as cur:
    40→            if args.all_languages:
    41→                base = """
    42→                    select youtube_id
    43→                    from videos
    44→                    where r2_tar_key is not null
    45→                      and status = 'COMPLETED'
    46→                """
    47→                params = []
    48→            else:
    49→                if not args.language:
    50→                    raise SystemExit("Provide --language or use --all-languages")
    51→                base = """
    52→                    select youtube_id
    53→                    from videos
    54→                    where lower(language) = %s
    55→                      and r2_tar_key is not null
    56→                      and status = 'COMPLETED'
    57→                """
    58→                params = [args.language.lower()]
    59→
    60→            if args.limit:
    61→                base += " limit %s"
    62→                params.append(args.limit)
    63→
    64→            cur.execute(f"select count(*) from ({base}) as t", params)
    65→            total = cur.fetchone()[0]
    66→
    67→    if args.dry_run:
    68→        if args.all_languages:
    69→            print(f"Would insert {total} rows for all languages")
    70→        else:
    71→            print(f"Would insert {total} rows for language={args.language}")
    72→        return
    73→
    74→    with psycopg.connect(db_url) as conn:
    75→        with conn.cursor() as cur:
    76→            if args.all_languages:
    77→                query = """
    78→                    insert into video_tasks (video_id, language, r2_tar_key)
    79→                    select youtube_id, lower(language), r2_tar_key
    80→                    from videos
    81→                    where r2_tar_key is not null
    82→                      and status = 'COMPLETED'
    83→                """
    84→                params = []
    85→            else:
    86→                query = """
    87→                    insert into video_tasks (video_id, language, r2_tar_key)
    88→                    select youtube_id, lower(language), r2_tar_key
    89→                    from videos
    90→                    where lower(language) = %s
    91→                      and r2_tar_key is not null
    92→                      and status = 'COMPLETED'
    93→                """
    94→                params = [args.language.lower()]
    95→
    96→            if args.limit:
    97→                query += " limit %s"
    98→                params.append(args.limit)
    99→
   100→            query += " on conflict (video_id) do nothing"
   101→            cur.execute(query, params)
   102→        conn.commit()
   103→
   104→    if args.all_languages:
   105→        print(f"Inserted {total} rows for all languages")
   106→    else:
   107→        print(f"Inserted {total} rows for language={args.language}")
   108→
   109→
   110→if __name__ == "__main__":
   111→    main()
   112→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

#!/usr/bin/env python3
"""Seed video_tasks from the existing videos table (Supabase/Postgres)."""

from __future__ import annotations

import argparse
from pathlib import Path

import psycopg
import sys

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))

from prepare_shards import load_env

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Seed video_tasks table.")
    parser.add_argument("--language")
    parser.add_argument(
        "--all-languages",
        action="store_true",
        help="Seed tasks for all languages (ignores --language)",
    )
    parser.add_argument("--limit", type=int, default=None)
    parser.add_argument("--dry-run", action="store_true")
    return parser.parse_args()

def main() -> None:
    args = parse_args()
    env = load_env(Path(".env"))
    db_url = env.get("DATABASE_URL")
    if not db_url:
        raise SystemExit("DATABASE_URL missing in .env")

    with psycopg.connect(db_url) as conn:
        with conn.cursor() as cur:
            if args.all_languages:
                base = """
                    select youtube_id
                    from videos
                    where r2_tar_key is not null
                      and status = 'COMPLETED'
                """
                params = []
            else:
                if not args.language:
                    raise SystemExit("Provide --language or use --all-languages")
                base = """
                    select youtube_id
                    from videos
                    where lower(language) = %s
                      and r2_tar_key is not null
                      and status = 'COMPLETED'
                """
                params = [args.language.lower()]

            if args.limit:
                base += " limit %s"
                params.append(args.limit)

            cur.execute(f"select count(*) from ({base}) as t", params)
            total = cur.fetchone()[0]

    if args.dry_run:
        if args.all_languages:
            print(f"Would insert {total} rows for all languages")
        else:
            print(f"Would insert {total} rows for language={args.language}")
        return

    with psycopg.connect(db_url) as conn:
        with conn.cursor() as cur:
            if args.all_languages:
                query = """
                    insert into video_tasks (video_id, language, r2_tar_key)
                    select youtube_id, lower(language), r2_tar_key
                    from videos
                    where r2_tar_key is not null
                      and status = 'COMPLETED'
                """
                params = []
            else:
                query = """
                    insert into video_tasks (video_id, language, r2_tar_key)
                    select youtube_id, lower(language), r2_tar_key
                    from videos
                    where lower(language) = %s
                      and r2_tar_key is not null
                      and status = 'COMPLETED'
                """
                params = [args.language.lower()]

            if args.limit:
                query += " limit %s"
                params.append(args.limit)

            query += " on conflict (video_id) do nothing"
            cur.execute(query, params)
        conn.commit()

    if args.all_languages:
        print(f"Inserted {total} rows for all languages")
    else:
        print(f"Inserted {total} rows for language={args.language}")

if __name__ == "__main__":
    main()


     1→#!/usr/bin/env python3
     2→"""Release stale claims from crashed workers.
     3→
     4→This script should be run periodically (e.g., via cron) to:
     5→1. Release video_tasks with expired leases back to 'pending'
     6→2. Mark associated shard_runs as 'stale'
     7→3. Log recovery actions for debugging
     8→
     9→Usage:
    10→    python release_stale_claims.py [--dry-run] [--env-file .env]
    11→"""
    12→
    13→from __future__ import annotations
    14→
    15→import argparse
    16→import logging
    17→import sys
    18→from datetime import datetime
    19→from pathlib import Path
    20→
    21→import psycopg
    22→
    23→if __package__ in (None, ""):
    24→    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
    25→
    26→from workers.config import EnvConfig
    27→
    28→logging.basicConfig(
    29→    level=logging.INFO,
    30→    format="%(asctime)s [%(levelname)s] %(message)s",
    31→    datefmt="%Y-%m-%d %H:%M:%S",
    32→)
    33→logger = logging.getLogger("stale-recovery")
    34→
    35→
    36→def release_stale_video_tasks(conn, dry_run: bool = False) -> int:
    37→    """Release video_tasks with expired leases back to pending."""
    38→    with conn.cursor() as cur:
    39→        if dry_run:
    40→            cur.execute(
    41→                """
    42→                select video_id, claimed_by, lease_expires_at, status
    43→                from video_tasks
    44→                where status in ('claimed', 'processing')
    45→                  and lease_expires_at < now()
    46→                """
    47→            )
    48→            stale = cur.fetchall()
    49→            for row in stale:
    50→                logger.info(
    51→                    f"[DRY-RUN] Would release video {row[0]} "
    52→                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
    53→                )
    54→            return len(stale)
    55→        else:
    56→            cur.execute(
    57→                """
    58→                update video_tasks
    59→                set status = 'pending',
    60→                    claimed_by = null,
    61→                    lease_expires_at = null,
    62→                    updated_at = now()
    63→                where status in ('claimed', 'processing')
    64→                  and lease_expires_at < now()
    65→                returning video_id, claimed_by
    66→                """
    67→            )
    68→            released = cur.fetchall()
    69→            for video_id, claimed_by in released:
    70→                logger.info(f"Released video {video_id} (was claimed by {claimed_by})")
    71→            return len(released)
    72→
    73→
    74→def mark_stale_shards(conn, dry_run: bool = False) -> int:
    75→    """Mark shard_runs with expired leases as stale."""
    76→    with conn.cursor() as cur:
    77→        if dry_run:
    78→            cur.execute(
    79→                """
    80→                select shard_id, claimed_by, lease_expires_at, status
    81→                from shard_runs
    82→                where status in ('claimed', 'processing', 'uploading')
    83→                  and lease_expires_at < now()
    84→                """
    85→            )
    86→            stale = cur.fetchall()
    87→            for row in stale:
    88→                logger.info(
    89→                    f"[DRY-RUN] Would mark shard {row[0]} as stale "
    90→                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
    91→                )
    92→            return len(stale)
    93→        else:
    94→            cur.execute(
    95→                """
    96→                update shard_runs
    97→                set status = 'stale',
    98→                    updated_at = now()
    99→                where status in ('claimed', 'processing', 'uploading')
   100→                  and lease_expires_at < now()
   101→                returning shard_id, claimed_by
   102→                """
   103→            )
   104→            stale = cur.fetchall()
   105→            for shard_id, claimed_by in stale:
   106→                logger.info(f"Marked shard {shard_id} as stale (was claimed by {claimed_by})")
   107→            return len(stale)
   108→
   109→
   110→def release_stale_languages(conn, dry_run: bool = False) -> int:
   111→    """Release language_queue entries with expired leases back to pending."""
   112→    with conn.cursor() as cur:
   113→        if dry_run:
   114→            cur.execute(
   115→                """
   116→                select language, claimed_by, lease_expires_at, status
   117→                from language_queue
   118→                where status = 'processing'
   119→                  and lease_expires_at < now()
   120→                """
   121→            )
   122→            stale = cur.fetchall()
   123→            for row in stale:
   124→                logger.info(
   125→                    f"[DRY-RUN] Would release language {row[0]} "
   126→                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
   127→                )
   128→            return len(stale)
   129→        else:
   130→            cur.execute(
   131→                """
   132→                update language_queue
   133→                set status = 'pending',
   134→                    claimed_by = null,
   135→                    lease_expires_at = null,
   136→                    updated_at = now()
   137→                where status = 'processing'
   138→                  and lease_expires_at < now()
   139→                returning language, claimed_by
   140→                """
   141→            )
   142→            released = cur.fetchall()
   143→            for language, claimed_by in released:
   144→                logger.info(
   145→                    f"Released language {language} (was claimed by {claimed_by})"
   146→                )
   147→            if released:
   148→                cur.execute(
   149→                    """
   150→                    update language_state
   151→                    set active_language = null,
   152→                        updated_at = now()
   153→                    where id = 1
   154→                      and active_language = any(%s)
   155→                    """,
   156→                    ([row[0] for row in released],),
   157→                )
   158→            return len(released)
   159→
   160→
   161→def log_recovery_event(conn, video_ids: int, shard_ids: int, language_ids: int) -> None:
   162→    """Log recovery event for audit trail."""
   163→    with conn.cursor() as cur:
   164→        cur.execute(
   165→            """
   166→            insert into video_task_events (event, details)
   167→            values ('stale_recovery', %s)
   168→            """,
   169→            (
   170→                f'{{"videos_released": {video_ids}, '
   171→                f'"shards_marked_stale": {shard_ids}, '
   172→                f'"languages_released": {language_ids}}}',
   173→            ),
   174→        )
   175→
   176→
   177→def get_stats(conn) -> dict:
   178→    """Get current task statistics."""
   179→    with conn.cursor() as cur:
   180→        cur.execute(
   181→            """
   182→            select status, count(*)
   183→            from video_tasks
   184→            group by status
   185→            """
   186→        )
   187→        video_stats = dict(cur.fetchall())
   188→
   189→        cur.execute(
   190→            """
   191→            select status, count(*)
   192→            from shard_runs
   193→            group by status
   194→            """
   195→        )
   196→        shard_stats = dict(cur.fetchall())
   197→
   198→        cur.execute(
   199→            """
   200→            select status, count(*)
   201→            from language_queue
   202→            group by status
   203→            """
   204→        )
   205→        language_stats = dict(cur.fetchall())
   206→
   207→        cur.execute(
   208→            """
   209→            select count(distinct worker_id)
   210→            from worker_heartbeats_v2
   211→            where last_heartbeat > now() - interval '5 minutes'
   212→            """
   213→        )
   214→        active_workers = cur.fetchone()[0]
   215→
   216→    return {
   217→        "video_tasks": video_stats,
   218→        "shard_runs": shard_stats,
   219→        "language_queue": language_stats,
   220→        "active_workers": active_workers,
   221→    }
   222→
   223→
   224→def main():
   225→    parser = argparse.ArgumentParser(description="Release stale claims from crashed workers")
   226→    parser.add_argument("--dry-run", action="store_true", help="Show what would be done")
   227→    parser.add_argument("--env-file", type=Path, default=Path(".env"))
   228→    parser.add_argument("--stats-only", action="store_true", help="Only show stats, don't recover")
   229→    args = parser.parse_args()
   230→
   231→    # Load config
   232→    if args.env_file.exists():
   233→        env_config = EnvConfig.from_file(args.env_file)
   234→    else:
   235→        env_config = EnvConfig.from_env_vars()
   236→
   237→    if not env_config.database_url:
   238→        logger.error("DATABASE_URL not configured")
   239→        raise SystemExit(1)
   240→
   241→    with psycopg.connect(env_config.database_url) as conn:
   242→        # Show current stats
   243→        stats = get_stats(conn)
   244→        logger.info("=== Current Statistics ===")
   245→        logger.info(f"Video tasks: {stats['video_tasks']}")
   246→        logger.info(f"Shard runs: {stats['shard_runs']}")
   247→        logger.info(f"Active workers (last 5 min): {stats['active_workers']}")
   248→
   249→        if args.stats_only:
   250→            return
   251→
   252→        logger.info("")
   253→        logger.info("=== Releasing Stale Claims ===")
   254→
   255→        # Release stale claims
   256→        videos_released = release_stale_video_tasks(conn, args.dry_run)
   257→        shards_marked = mark_stale_shards(conn, args.dry_run)
   258→        languages_released = release_stale_languages(conn, args.dry_run)
   259→
   260→        if not args.dry_run:
   261→            if videos_released > 0 or shards_marked > 0 or languages_released > 0:
   262→                log_recovery_event(conn, videos_released, shards_marked, languages_released)
   263→            conn.commit()
   264→
   265→        logger.info("")
   266→        logger.info(f"Released {videos_released} video tasks")
   267→        logger.info(f"Marked {shards_marked} shards as stale")
   268→        logger.info(f"Released {languages_released} languages")
   269→
   270→        # Show updated stats
   271→        if not args.dry_run and (videos_released > 0 or shards_marked > 0):
   272→            stats = get_stats(conn)
   273→            logger.info("")
   274→            logger.info("=== Updated Statistics ===")
   275→            logger.info(f"Video tasks: {stats['video_tasks']}")
   276→            logger.info(f"Shard runs: {stats['shard_runs']}")
   277→
   278→
   279→if __name__ == "__main__":
   280→    main()
   281→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

#!/usr/bin/env python3
"""Release stale claims from crashed workers.

This script should be run periodically (e.g., via cron) to:
1. Release video_tasks with expired leases back to 'pending'
2. Mark associated shard_runs as 'stale'
3. Log recovery actions for debugging

Usage:
    python release_stale_claims.py [--dry-run] [--env-file .env]
"""

from __future__ import annotations

import argparse
import logging
import sys
from datetime import datetime
from pathlib import Path

import psycopg

if __package__ in (None, ""):
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from workers.config import EnvConfig

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger("stale-recovery")

def release_stale_video_tasks(conn, dry_run: bool = False) -> int:
    """Release video_tasks with expired leases back to pending."""
    with conn.cursor() as cur:
        if dry_run:
            cur.execute(
                """
                select video_id, claimed_by, lease_expires_at, status
                from video_tasks
                where status in ('claimed', 'processing')
                  and lease_expires_at < now()
                """
            )
            stale = cur.fetchall()
            for row in stale:
                logger.info(
                    f"[DRY-RUN] Would release video {row[0]} "
                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
                )
            return len(stale)
        else:
            cur.execute(
                """
                update video_tasks
                set status = 'pending',
                    claimed_by = null,
                    lease_expires_at = null,
                    updated_at = now()
                where status in ('claimed', 'processing')
                  and lease_expires_at < now()
                returning video_id, claimed_by
                """
            )
            released = cur.fetchall()
            for video_id, claimed_by in released:
                logger.info(f"Released video {video_id} (was claimed by {claimed_by})")
            return len(released)

def mark_stale_shards(conn, dry_run: bool = False) -> int:
    """Mark shard_runs with expired leases as stale."""
    with conn.cursor() as cur:
        if dry_run:
            cur.execute(
                """
                select shard_id, claimed_by, lease_expires_at, status
                from shard_runs
                where status in ('claimed', 'processing', 'uploading')
                  and lease_expires_at < now()
                """
            )
            stale = cur.fetchall()
            for row in stale:
                logger.info(
                    f"[DRY-RUN] Would mark shard {row[0]} as stale "
                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
                )
            return len(stale)
        else:
            cur.execute(
                """
                update shard_runs
                set status = 'stale',
                    updated_at = now()
                where status in ('claimed', 'processing', 'uploading')
                  and lease_expires_at < now()
                returning shard_id, claimed_by
                """
            )
            stale = cur.fetchall()
            for shard_id, claimed_by in stale:
                logger.info(f"Marked shard {shard_id} as stale (was claimed by {claimed_by})")
            return len(stale)

def release_stale_languages(conn, dry_run: bool = False) -> int:
    """Release language_queue entries with expired leases back to pending."""
    with conn.cursor() as cur:
        if dry_run:
            cur.execute(
                """
                select language, claimed_by, lease_expires_at, status
                from language_queue
                where status = 'processing'
                  and lease_expires_at < now()
                """
            )
            stale = cur.fetchall()
            for row in stale:
                logger.info(
                    f"[DRY-RUN] Would release language {row[0]} "
                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
                )
            return len(stale)
        else:
            cur.execute(
                """
                update language_queue
                set status = 'pending',
                    claimed_by = null,
                    lease_expires_at = null,
                    updated_at = now()
                where status = 'processing'
                  and lease_expires_at < now()
                returning language, claimed_by
                """
            )
            released = cur.fetchall()
            for language, claimed_by in released:
                logger.info(
                    f"Released language {language} (was claimed by {claimed_by})"
                )
            if released:
                cur.execute(
                    """
                    update language_state
                    set active_language = null,
                        updated_at = now()
                    where id = 1
                      and active_language = any(%s)
                    """,
                    ([row[0] for row in released],),
                )
            return len(released)

def log_recovery_event(conn, video_ids: int, shard_ids: int, language_ids: int) -> None:
    """Log recovery event for audit trail."""
    with conn.cursor() as cur:
        cur.execute(
            """
            insert into video_task_events (event, details)
            values ('stale_recovery', %s)
            """,
            (
                f'{{"videos_released": {video_ids}, '
                f'"shards_marked_stale": {shard_ids}, '
                f'"languages_released": {language_ids}}}',
            ),
        )

def get_stats(conn) -> dict:
    """Get current task statistics."""
    with conn.cursor() as cur:
        cur.execute(
            """
            select status, count(*)
            from video_tasks
            group by status
            """
        )
        video_stats = dict(cur.fetchall())

        cur.execute(
            """
            select status, count(*)
            from shard_runs
            group by status
            """
        )
        shard_stats = dict(cur.fetchall())

        cur.execute(
            """
            select status, count(*)
            from language_queue
            group by status
            """
        )
        language_stats = dict(cur.fetchall())

        cur.execute(
            """
            select count(distinct worker_id)
            from worker_heartbeats_v2
            where last_heartbeat > now() - interval '5 minutes'
            """
        )
        active_workers = cur.fetchone()[0]

    return {
        "video_tasks": video_stats,
        "shard_runs": shard_stats,
        "language_queue": language_stats,
        "active_workers": active_workers,
    }

def main():
    parser = argparse.ArgumentParser(description="Release stale claims from crashed workers")
    parser.add_argument("--dry-run", action="store_true", help="Show what would be done")
    parser.add_argument("--env-file", type=Path, default=Path(".env"))
    parser.add_argument("--stats-only", action="store_true", help="Only show stats, don't recover")
    args = parser.parse_args()

    # Load config
    if args.env_file.exists():
        env_config = EnvConfig.from_file(args.env_file)
    else:
        env_config = EnvConfig.from_env_vars()

    if not env_config.database_url:
        logger.error("DATABASE_URL not configured")
        raise SystemExit(1)

    with psycopg.connect(env_config.database_url) as conn:
        # Show current stats
        stats = get_stats(conn)
        logger.info("=== Current Statistics ===")
        logger.info(f"Video tasks: {stats['video_tasks']}")
        logger.info(f"Shard runs: {stats['shard_runs']}")
        logger.info(f"Active workers (last 5 min): {stats['active_workers']}")

        if args.stats_only:
            return

        logger.info("")
        logger.info("=== Releasing Stale Claims ===")

        # Release stale claims
        videos_released = release_stale_video_tasks(conn, args.dry_run)
        shards_marked = mark_stale_shards(conn, args.dry_run)
        languages_released = release_stale_languages(conn, args.dry_run)

        if not args.dry_run:
            if videos_released > 0 or shards_marked > 0 or languages_released > 0:
                log_recovery_event(conn, videos_released, shards_marked, languages_released)
            conn.commit()

        logger.info("")
        logger.info(f"Released {videos_released} video tasks")
        logger.info(f"Marked {shards_marked} shards as stale")
        logger.info(f"Released {languages_released} languages")

        # Show updated stats
        if not args.dry_run and (videos_released > 0 or shards_marked > 0):
            stats = get_stats(conn)
            logger.info("")
            logger.info("=== Updated Statistics ===")
            logger.info(f"Video tasks: {stats['video_tasks']}")
            logger.info(f"Shard runs: {stats['shard_runs']}")

if __name__ == "__main__":
    main()


Let me search for any other metrics or monitoring-related files:


Grep

insert into|insert into metrics|update.*metrics


Bash

List all worker files


total 200
drwxrwxr-x 3 ubuntu ubuntu  4096 Jan 20 05:40 .
drwxrwxr-x 9 ubuntu ubuntu  4096 Jan 20 05:23 ..
-rw-rw-r-- 1 ubuntu ubuntu   512 Jan 20 04:00 claim_shard.sql
-rw-rw-r-- 1 ubuntu ubuntu  6596 Jan 20 05:38 config.py
-rw-rw-r-- 1 ubuntu ubuntu  1767 Jan 20 05:41 docker-compose.yml
-rw-rw-r-- 1 ubuntu ubuntu  1069 Jan 20 04:14 Dockerfile.cpu
-rw-rw-r-- 1 ubuntu ubuntu   322 Jan 20 04:01 Dockerfile.gpu
-rwxrwxr-x 1 ubuntu ubuntu  3552 Jan 20 05:39 entrypoint.sh
-rw-rw-r-- 1 ubuntu ubuntu    79 Jan 20 04:14 __init__.py
drwxrwxr-x 2 ubuntu ubuntu  4096 Jan 20 05:43 __pycache__
-rw-rw-r-- 1 ubuntu ubuntu 12553 Jan 20 04:08 r2_uploader.py
-rw-rw-r-- 1 ubuntu ubuntu 12404 Jan 20 05:43 README.md
-rw-rw-r-- 1 ubuntu ubuntu  9306 Jan 20 05:41 release_stale_claims.py
-rw-rw-r-- 1 ubuntu ubuntu   258 Jan 20 04:00 release_stale.sql
-rw-rw-r-- 1 ubuntu ubuntu   178 Jan 20 04:12 requirements.txt
-rw-rw-r-- 1 ubuntu ubuntu  2363 Jan 20 05:38 schema.sql
-rw-rw-r-- 1 ubuntu ubuntu  2300 Jan 20 05:40 seed_language_queue.py
-rw-rw-r-- 1 ubuntu ubuntu  3544 Jan 20 05:43 seed_video_tasks.py
-rw-rw-r-- 1 ubuntu ubuntu 11939 Jan 20 04:47 storage_manager.py
-rw-rw-r-- 1 ubuntu ubuntu  7671 Jan 20 05:40 worker_plan.md
-rw-rw-r-- 1 ubuntu ubuntu 32578 Jan 20 04:40 worker_processing.py
-rw-rw-r-- 1 ubuntu ubuntu 37289 Jan 20 05:39 worker.py


workers/worker.py:226:                insert into shard_runs (
workers/worker.py:454:                insert into worker_heartbeats_v2 (
workers/worker.py:836:                self.update_shard_status(conn, shard_id, "completed", metrics=stats.to_dict())
workers/seed_video_tasks.py:78:                    insert into video_tasks (video_id, language, r2_tar_key)
workers/seed_video_tasks.py:87:                    insert into video_tasks (video_id, language, r2_tar_key)
workers/release_stale_claims.py:166:            insert into video_task_events (event, details)
workers/seed_language_queue.py:63:                    insert into language_queue (language, priority)


Perfect! Let me create a comprehensive report based on all the findings:


grep -r "video_task_events\|metrics\|ProcessingStats" /home/ubuntu/maya3_prepare/workers/ --include="*.py" | head -30

Find additional metrics usage


/home/ubuntu/maya3_prepare/workers/worker_processing.py:- Compute audio metrics (LUFS, spectral, etc.)
/home/ubuntu/maya3_prepare/workers/worker_processing.py:class ProcessingStats:
/home/ubuntu/maya3_prepare/workers/worker_processing.py:def compute_spectral_metrics(audio: np.ndarray, sr: int) -> tuple[float, float, float]:
/home/ubuntu/maya3_prepare/workers/worker_processing.py:    """Compute spectral metrics: rolloff, effective nyquist, high freq ratio."""
/home/ubuntu/maya3_prepare/workers/worker_processing.py:def compute_audio_metrics(audio: np.ndarray, sr: int) -> dict[str, float]:
/home/ubuntu/maya3_prepare/workers/worker_processing.py:    """Compute all audio metrics for a segment."""
/home/ubuntu/maya3_prepare/workers/worker_processing.py:    rolloff_hz, effective_nyquist_hz, high_freq_ratio = compute_spectral_metrics(audio, sr)
/home/ubuntu/maya3_prepare/workers/worker_processing.py:def compute_quality_score(metrics: dict[str, float], has_music: bool) -> float:
/home/ubuntu/maya3_prepare/workers/worker_processing.py:        0.3 * (1.0 - metrics["clipping_fraction"])
/home/ubuntu/maya3_prepare/workers/worker_processing.py:        + 0.2 * normalize(metrics["snr_proxy"], 10, 40)
/home/ubuntu/maya3_prepare/workers/worker_processing.py:        + 0.2 * normalize(metrics["loudness_lufs"], -40, -14)
/home/ubuntu/maya3_prepare/workers/worker_processing.py:        + 0.1 * (1.0 if abs(metrics["dc_offset"]) < 0.01 else 0.5)
/home/ubuntu/maya3_prepare/workers/worker_processing.py:        metrics: dict[str, float],
/home/ubuntu/maya3_prepare/workers/worker_processing.py:        if metrics["clipping_fraction"] > 0.1:
/home/ubuntu/maya3_prepare/workers/worker_processing.py:        quality_score = compute_quality_score(metrics, has_music)
/home/ubuntu/maya3_prepare/workers/worker_processing.py:        effective_nyquist = metrics["effective_nyquist_hz"]
/home/ubuntu/maya3_prepare/workers/worker_processing.py:            "spectral_rolloff_hz": metrics["spectral_rolloff_hz"],
/home/ubuntu/maya3_prepare/workers/worker_processing.py:            "high_freq_energy_ratio": metrics["high_freq_energy_ratio"],
/home/ubuntu/maya3_prepare/workers/worker_processing.py:            "loudness_lufs": metrics["loudness_lufs"],
/home/ubuntu/maya3_prepare/workers/worker_processing.py:            "loudness_rms_db": metrics["loudness_rms_db"],
/home/ubuntu/maya3_prepare/workers/worker_processing.py:            "clipping_fraction": metrics["clipping_fraction"],
/home/ubuntu/maya3_prepare/workers/worker_processing.py:            "snr_proxy": metrics["snr_proxy"],
/home/ubuntu/maya3_prepare/workers/worker_processing.py:        slice_metrics = compute_audio_metrics(audio_slice, sr)
/home/ubuntu/maya3_prepare/workers/worker_processing.py:            slice_metrics,
/home/ubuntu/maya3_prepare/workers/worker_processing.py:    full_metrics = compute_audio_metrics(audio, sr)
/home/ubuntu/maya3_prepare/workers/worker_processing.py:    effective_nyquist = full_metrics["effective_nyquist_hz"]
/home/ubuntu/maya3_prepare/workers/worker_processing.py:                slice_metrics = compute_audio_metrics(audio_slice, sr)
/home/ubuntu/maya3_prepare/workers/worker_processing.py:                slice_effective_nyquist = slice_metrics["effective_nyquist_hz"]
/home/ubuntu/maya3_prepare/workers/worker_processing.py:                    slice_metrics,
/home/ubuntu/maya3_prepare/workers/worker_processing.py:                full_metrics,


Perfect! Now I have all the information. Let me create a comprehensive report:

## Supabase Metrics & Analytics Exploration Report

I've completed a thorough exploration of the codebase. Here's a comprehensive summary of all Supabase integrations, metrics tracking, and database schemas:

---

## 1. **Supabase Database Connection**

**Location:** `/home/ubuntu/maya3_prepare/workers/config.py`

- Uses **`psycopg`** library for PostgreSQL connections
- Database URL from `.env` file: `DATABASE_URL`
- Connected via standard PostgreSQL protocol (compatible with Supabase)

---

## 2. **Core Tables Being Used**

### **video_tasks** (Main task queue)
**Location:** `/home/ubuntu/maya3_prepare/workers/schema.sql` (lines 3-15)

Fields tracked:
- `video_id` (TEXT, PRIMARY KEY) - Unique video identifier
- `language` (TEXT, NOT NULL) - Language of the video
- `r2_tar_key` (TEXT, NOT NULL) - R2 storage location of tar file
- `status` (TEXT) - Pipeline status: `pending|claimed|processing|uploading|completed|failed`
- `shard_id` (UUID) - Reference to shard being processed
- `claimed_by` (TEXT) - Worker ID that claimed the task
- `lease_expires_at` (TIMESTAMPTZ) - Lease expiration for crash recovery
- `attempts` (INT) - Number of processing attempts
- `last_error` (TEXT) - Latest error message
- `created_at` (TIMESTAMPTZ) - Task creation timestamp
- `updated_at` (TIMESTAMPTZ) - Last update timestamp

**Index:** `video_tasks_language_status_idx` on `(language, status)` for efficient lookups

---

### **shard_runs** (Processing batches)
**Location:** `/home/ubuntu/maya3_prepare/workers/schema.sql` (lines 20-33)

Fields tracked:
- `shard_id` (UUID, PRIMARY KEY) - Unique shard identifier
- `language` (TEXT, NOT NULL) - Language being processed
- `target_video_count` (INT) - Expected number of videos in shard
- `assigned_video_count` (INT) - Actual videos assigned to shard
- `status` (TEXT) - `claimed|processing|uploading|completed|failed|stale`
- `claimed_by` (TEXT) - Worker ID processing this shard
- `lease_expires_at` (TIMESTAMPTZ) - When lease expires
- `shard_prefix` (TEXT) - R2 path prefix like `"language/shard-uuid"`
- `output_format` (TEXT) - `flac` or `wav`
- **`metrics` (JSONB)** - Processing statistics stored as JSON
- `created_at` (TIMESTAMPTZ)
- `updated_at` (TIMESTAMPTZ)

**Index:** `shard_runs_language_status_idx` on `(language, status)`

**Metrics Stored in JSONB:**
```python
# From worker_processing.py ProcessingStats.to_dict()
{
    "videos": int,
    "segments_total": int,
    "segments_kept": {16000: int, 24000: int, 32000: int, 44100: int},
    "dropped_too_short": int,
    "dropped_too_long": int,
    "dropped_clipped": int,
    "segments_chunked": int,
    "chunks_emitted": int,
    "native_sr_buckets": {
        "<16k": int,
        "16-24k": int,
        "24-32k": int,
        "32-44k": int,
        ">=44k": int
    }
}
```

---

### **worker_heartbeats_v2** (Worker telemetry)
**Location:** `/home/ubuntu/maya3_prepare/workers/schema.sql` (lines 38-51) and `/home/ubuntu/maya3_prepare/workers/worker.py` (lines 446-486)

Fields tracked:
- `worker_id` (TEXT, PRIMARY KEY) - Unique worker identifier
- `hostname` (TEXT) - Worker hostname
- `instance_type` (TEXT) - VM instance type (e.g., GPU model)
- `language` (TEXT) - Currently processing language
- `status` (TEXT) - Worker state: `idle|claiming|processing|uploading|failed`
- `shard_id` (UUID) - Current shard ID being processed
- `videos_assigned` (INT) - Total videos assigned in current batch
- `videos_processed` (INT) - Videos processed so far
- `segments_processed` (INT) - Audio segments extracted
- `outputs_generated` (INT) - Final output samples created
- `last_error` (TEXT) - Last error encountered
- `last_heartbeat` (TIMESTAMPTZ) - Timestamp of this heartbeat

**Heartbeat Interval:** Every 15 seconds (configurable via `HEARTBEAT_INTERVAL_SEC`)

**Location of send_heartbeat():** `/home/ubuntu/maya3_prepare/workers/worker.py` lines 446-486

---

### **language_queue** (Language scheduling)
**Location:** `/home/ubuntu/maya3_prepare/workers/schema.sql` (lines 63-72)

Fields tracked:
- `language` (TEXT, PRIMARY KEY) - Language code
- `priority` (INT) - Processing priority (video count)
- `status` (TEXT) - `pending|processing|completed|stale`
- `claimed_by` (TEXT) - Worker currently processing this language
- `lease_expires_at` (TIMESTAMPTZ) - Lease expiration
- `created_at` (TIMESTAMPTZ)
- `updated_at` (TIMESTAMPTZ)
- `completed_at` (TIMESTAMPTZ) - When language finished

---

### **language_state** (Global state)
**Location:** `/home/ubuntu/maya3_prepare/workers/schema.sql` (lines 74-78)

Fields tracked:
- `id` (INT, PRIMARY KEY) - Always 1 (singleton table)
- `active_language` (TEXT) - Current language being processed
- `updated_at` (TIMESTAMPTZ)

---

### **video_task_events** (Audit log)
**Location:** `/home/ubuntu/maya3_prepare/workers/schema.sql` (lines 53-61)

Fields tracked:
- `id` (BIGSERIAL, PRIMARY KEY) - Event sequence number
- `video_id` (TEXT) - Associated video
- `shard_id` (UUID) - Associated shard
- `worker_id` (TEXT) - Worker that triggered event
- `event` (TEXT) - Event type (e.g., `stale_recovery`)
- `details` (JSONB) - Event-specific data
- `created_at` (TIMESTAMPTZ) - Event timestamp

---

## 3. **Metrics Being Tracked**

### **Per-Segment Audio Metrics** (in segment metadata)
**Location:** `/home/ubuntu/maya3_prepare/workers/worker_processing.py` (lines 194-224)

For each audio segment, these metrics are computed:
- `loudness_lufs` (FLOAT) - Integrated loudness in LUFS
- `loudness_rms_db` (FLOAT) - RMS loudness in dB
- `clipping_fraction` (FLOAT) - Percentage of clipped samples (>0.99)
- `dc_offset` (FLOAT) - DC component of signal
- `snr_proxy` (FLOAT) - Signal-to-noise ratio proxy
- `spectral_rolloff_hz` (FLOAT) - Frequency containing 85% of energy
- `effective_nyquist_hz` (FLOAT) - Effective bandwidth
- `high_freq_energy_ratio` (FLOAT) - Energy above 8kHz

### **Per-Segment Output Metadata**
**Location:** `/home/ubuntu/maya3_prepare/workers/worker_processing.py` (lines 495-532)

Tracked fields:
- `id` - Sample ID
- `video_id` - Source video
- `language` - Language code
- `segment_index` - Segment ordinal
- `chunk_index` - Chunk within segment (if chunked)
- `start`, `end` - Time in segment
- `original_start`, `original_end` - Original timestamps
- `duration_sec` - Duration
- `native_sr` - Native sample rate
- `effective_nyquist_hz` - Bandwidth classification
- `spectral_rolloff_hz` - Spectral metric
- `high_freq_energy_ratio` - High frequency content
- `is_truly_wideband` - Bool (effective_nyquist >= 12kHz)
- `native_sr_bucket` - Bandwidth category
- `loudness_lufs`, `loudness_rms_db` - Loudness metrics
- `clipping_fraction` - Clipping metric
- `snr_proxy` - SNR metric
- `quality_score` - Overall quality 0-1
- `music_prob` - Music probability
- `music_stats` - Detailed music statistics
- `speaker` - Speaker ID
- `vad_speech_regions` - Speech detection regions
- `duration_bucket` - Duration category
- `sampling_weight` - Sampling probability

### **Worker State Metrics**
**Location:** `/home/ubuntu/maya3_prepare/workers/worker.py` (lines 72-83)

- `status` - Worker state
- `shard_id` - Current shard
- `videos_assigned` - Videos in batch
- `videos_processed` - Progress count
- `segments_processed` - Segments extracted
- `outputs_generated` - Final outputs
- `last_error` - Error message
- `processing_start` - Start timestamp

---

## 4. **How Metrics Flow to Supabase**

### **Heartbeat Push** (Every 15 seconds)
**Location:** `/home/ubuntu/maya3_prepare/workers/worker.py` lines 446-486

```python
# Inserts/updates worker_heartbeats_v2
insert into worker_heartbeats_v2 (
    worker_id, hostname, instance_type, language, status,
    shard_id, videos_assigned, videos_processed,
    segments_processed, outputs_generated, last_error, last_heartbeat
) values (...)
on conflict (worker_id) do update set ...
```

### **Shard Status Update** (On completion)
**Location:** `/home/ubuntu/maya3_prepare/workers/worker.py` lines 292-310

```python
# Updates shard_runs with final metrics
update shard_runs
set status = %s,
    metrics = coalesce(%s, metrics),  # <-- METRICS STORED HERE
    updated_at = now()
where shard_id = %s
```

Called with: `self.update_shard_status(conn, shard_id, "completed", metrics=stats.to_dict())`

### **Task Status Updates** (Throughout pipeline)
**Location:** `/home/ubuntu/maya3_prepare/workers/worker.py` lines 252-290

Tracks: `status`, `shard_id`, `last_error`, `updated_at`

---

## 5. **Supabase Client Setup**

**Method:** Direct psycopg PostgreSQL connection
**Location:** `/home/ubuntu/maya3_prepare/workers/worker.py` line 492

```python
with psycopg.connect(self.env_config.database_url) as conn:
    self.send_heartbeat(conn)
    conn.commit()
```

**Configuration Files:**
- **Main config:** `/home/ubuntu/maya3_prepare/workers/config.py`
- **Schema creation:** `/home/ubuntu/maya3_prepare/workers/schema.sql`

---

## 6. **Recovery & Monitoring Infrastructure**

### **Stale Claim Recovery**
**Location:** `/home/ubuntu/maya3_prepare/workers/release_stale_claims.py`

Tracks lease expiration for crash recovery:
- Releases expired `video_tasks` (claimed/processing → pending)
- Marks expired `shard_runs` as "stale"
- Releases expired `language_queue` entries
- Logs recovery events to `video_task_events` table

---

## 7. **Data Aggregation Scripts**

### **Summary Statistics**
**Location:** `/home/ubuntu/maya3_prepare/scripts/supabase_summary.py`

Queries aggregated data:
- Language stats (normalized)
- Sample rate distribution
- Status breakdown
- Classification breakdown
- Audio hours totals

---

## Key Summary

| Aspect | Details |
|--------|---------|
| **Connection** | psycopg (PostgreSQL) via `DATABASE_URL` |
| **Metrics Table** | `shard_runs.metrics` (JSONB) + `worker_heartbeats_v2` (detailed) |
| **Heartbeat Table** | `worker_heartbeats_v2` (every 15 sec) |
| **Task Tracking** | `video_tasks` (status, language, attempts, errors) |
| **Batch Tracking** | `shard_runs` (videos, metrics, output format) |
| **Audit Logging** | `video_task_events` (event log for recovery) |
| **Language Queue** | `language_queue` (priority scheduling) + `language_state` (active language) |
| **Audio Quality Metrics** | 8 metrics per segment + quality score |
| **Worker Metrics** | 10 fields including progress, errors, current shard |
| **Push Frequency** | Heartbeat every 15 seconds, final metrics on shard completion |

All files and absolute paths are provided above for your reference.