     1→#!/usr/bin/env python3
     2→"""Distributed shard worker for horizontal scaling.
     3→
     4→This worker:
     5→1. Claims a batch of videos from Supabase (with row-level locking)
     6→2. Downloads video tars from R2
     7→3. Processes segments to generate shards
     8→4. Uploads shards to R2
     9→5. Updates status in Supabase
    10→6. Cleans up local storage
    11→7. Reports heartbeat and metrics
    12→
    13→Key safety features:
    14→- Deterministic shard_id (same videos = same shard = idempotent upload)
    15→- FOR UPDATE SKIP LOCKED prevents duplicate claims
    16→- Lease expiry allows crash recovery
    17→- Storage monitoring prevents disk exhaustion
    18→"""
    19→
    20→from __future__ import annotations
    21→
    22→import argparse
    23→import hashlib
    24→import json
    25→import logging
    26→import os
    27→import signal
    28→import shutil
    29→import socket
    30→import sys
    31→import threading
    32→import time
    33→import uuid
    34→from dataclasses import dataclass, field
    35→from pathlib import Path
    36→from typing import Any, Optional
    37→
    38→import psycopg
    39→from boto3.s3.transfer import TransferConfig
    40→
    41→# Setup logging
    42→logging.basicConfig(
    43→    level=logging.INFO,
    44→    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
    45→    datefmt="%Y-%m-%d %H:%M:%S",
    46→)
    47→logger = logging.getLogger("worker")
    48→
    49→if __package__ in (None, ""):
    50→    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
    51→
    52→# Import worker modules
    53→from workers.config import WorkerConfig, EnvConfig
    54→from workers.storage_manager import StorageManager, StorageStats
    55→from workers.r2_uploader import R2Uploader
    56→from workers.worker_processing import (
    57→    VideoProcessor,
    58→    ChunkConfig,
    59→    SilenceConfig,
    60→    ProcessingStats,
    61→)
    62→
    63→
    64→@dataclass(frozen=True)
    65→class TaskRow:
    66→    """A video task from the database."""
    67→    video_id: str
    68→    r2_tar_key: str
    69→    language: str
    70→
    71→
    72→@dataclass
    73→class WorkerState:
    74→    """Current worker state for heartbeat reporting."""
    75→    status: str = "idle"
    76→    shard_id: Optional[str] = None
    77→    videos_assigned: int = 0
    78→    videos_processed: int = 0
    79→    segments_processed: int = 0
    80→    outputs_generated: int = 0
    81→    last_error: Optional[str] = None
    82→    processing_start: Optional[float] = None
    83→
    84→
    85→class ShardWorker:
    86→    """Main worker class that orchestrates the processing pipeline."""
    87→
    88→    def __init__(
    89→        self,
    90→        config: WorkerConfig,
    91→        env_config: EnvConfig,
    92→    ):
    93→        self.config = config
    94→        self.env_config = env_config
    95→
    96→        # Generate worker ID if not provided
    97→        self.worker_id = config.worker_id or f"{socket.gethostname()}-{os.getpid()}"
    98→        logger.info(f"Worker ID: {self.worker_id}")
    99→
   100→        # Initialize R2 clients
   101→        self.source_uploader = R2Uploader(
   102→            endpoint_url=env_config.r2_endpoint_url,
   103→            access_key_id=env_config.r2_access_key_id,
   104→            secret_access_key=env_config.r2_secret_access_key,
   105→            bucket=env_config.r2_cleaned_bucket,
   106→            max_retries=config.max_retries,
   107→            retry_delay_sec=config.retry_delay_sec,
   108→            max_concurrency=config.download_workers,
   109→        )
   110→
   111→        self.shard_uploader = R2Uploader(
   112→            endpoint_url=env_config.r2_endpoint_url,
   113→            access_key_id=env_config.r2_access_key_id,
   114→            secret_access_key=env_config.r2_secret_access_key,
   115→            bucket=env_config.r2_bucket_shards,
   116→            max_retries=config.max_retries,
   117→            retry_delay_sec=config.retry_delay_sec,
   118→            max_concurrency=config.upload_workers,
   119→        )
   120→
   121→        # Initialize storage manager
   122→        self.storage_manager = StorageManager(
   123→            work_dir=config.work_dir,
   124→            output_dir=config.output_dir,
   125→            max_usage_gb=config.max_disk_usage_gb,
   126→            cleanup_threshold_gb=config.cleanup_threshold_gb,
   127→            check_interval_sec=config.cleanup_check_interval_sec,
   128→        )
   129→
   130→        # Worker state
   131→        self.state = WorkerState()
   132→        self._state_lock = threading.Lock()
   133→        self._stop_event = threading.Event()
   134→        self._heartbeat_thread: Optional[threading.Thread] = None
   135→        self._lease_thread: Optional[threading.Thread] = None
   136→        self._lease_stop_event = threading.Event()
   137→
   138→        # Setup signal handlers
   139→        signal.signal(signal.SIGTERM, self._handle_signal)
   140→        signal.signal(signal.SIGINT, self._handle_signal)
   141→
   142→    def _handle_signal(self, signum, frame):
   143→        """Handle shutdown signals gracefully."""
   144→        logger.info(f"Received signal {signum}, initiating graceful shutdown...")
   145→        self._stop_event.set()
   146→
   147→    def _update_state(self, **kwargs) -> None:
   148→        """Thread-safe state update."""
   149→        with self._state_lock:
   150→            for key, value in kwargs.items():
   151→                if hasattr(self.state, key):
   152→                    setattr(self.state, key, value)
   153→
   154→    def _get_state(self) -> dict:
   155→        """Get current state as dict."""
   156→        with self._state_lock:
   157→            return {
   158→                "status": self.state.status,
   159→                "shard_id": self.state.shard_id,
   160→                "videos_assigned": self.state.videos_assigned,
   161→                "videos_processed": self.state.videos_processed,
   162→                "segments_processed": self.state.segments_processed,
   163→                "outputs_generated": self.state.outputs_generated,
   164→                "last_error": self.state.last_error,
   165→            }
   166→
   167→    def deterministic_shard_id(self, language: str, video_ids: list[str]) -> uuid.UUID:
   168→        """Generate deterministic shard ID from language + sorted video IDs.
   169→
   170→        This ensures that re-processing the same videos produces the same shard_id,
   171→        making uploads idempotent (overwrites same R2 key instead of creating duplicates).
   172→        """
   173→        joined = f"{language}:" + ",".join(sorted(video_ids))
   174→        digest = hashlib.sha1(joined.encode("utf-8")).hexdigest()
   175→        return uuid.uuid5(uuid.NAMESPACE_URL, digest)
   176→
   177→    def claim_videos(self, conn, language: str) -> list[TaskRow]:
   178→        """Claim a batch of pending videos with row-level locking.
   179→
   180→        Uses FOR UPDATE SKIP LOCKED to prevent duplicate claims across workers.
   181→        """
   182→        with conn.cursor() as cur:
   183→            cur.execute(
   184→                """
   185→                with picked as (
   186→                    select video_id
   187→                    from video_tasks
   188→                    where language = %s
   189→                      and status = 'pending'
   190→                    order by video_id
   191→                    limit %s
   192→                    for update skip locked
   193→                )
   194→                update video_tasks
   195→                set status = 'claimed',
   196→                    claimed_by = %s,
   197→                    lease_expires_at = now() + (%s || ' minutes')::interval,
   198→                    attempts = attempts + 1,
   199→                    updated_at = now()
   200→                where video_id in (select video_id from picked)
   201→                returning video_id, r2_tar_key, language
   202→                """,
   203→                (
   204→                    language,
   205→                    self.config.target_videos,
   206→                    self.worker_id,
   207→                    str(self.config.lease_minutes),
   208→                ),
   209→            )
   210→            rows = cur.fetchall()
   211→        return [TaskRow(*row) for row in rows]
   212→
   213→    def create_shard_run(
   214→        self,
   215→        conn,
   216→        shard_id: uuid.UUID,
   217→        assigned_count: int,
   218→        status: str = "claimed",
   219→        language: Optional[str] = None,
   220→    ) -> None:
   221→        """Create shard_runs entry for tracking."""
   222→        run_language = language or self.config.language
   223→        with conn.cursor() as cur:
   224→            cur.execute(
   225→                """
   226→                insert into shard_runs (
   227→                    shard_id, language, target_video_count, assigned_video_count,
   228→                    status, claimed_by, lease_expires_at, shard_prefix, output_format
   229→                ) values (
   230→                    %s, %s, %s, %s,
   231→                    %s, %s, now() + (%s || ' minutes')::interval, %s, %s
   232→                )
   233→                on conflict (shard_id) do update set
   234→                    status = excluded.status,
   235→                    claimed_by = excluded.claimed_by,
   236→                    lease_expires_at = excluded.lease_expires_at,
   237→                    updated_at = now()
   238→                """,
   239→                (
   240→                    str(shard_id),
   241→                    run_language,
   242→                    self.config.target_videos,
   243→                    assigned_count,
   244→                    status,
   245→                    self.worker_id,
   246→                    str(self.config.lease_minutes),
   247→                    f"{run_language}/shard-{shard_id}",
   248→                    self.config.output_format,
   249→                ),
   250→            )
   251→
   252→    def update_task_status(
   253→        self,
   254→        conn,
   255→        video_ids: list[str],
   256→        status: str,
   257→        shard_id: Optional[uuid.UUID] = None,
   258→        error: Optional[str] = None,
   259→        clear_claim: bool = False,
   260→    ) -> None:
   261→        """Update status for video tasks."""
   262→        if not video_ids:
   263→            return
   264→        with conn.cursor() as cur:
   265→            if clear_claim:
   266→                cur.execute(
   267→                    """
   268→                    update video_tasks
   269→                    set status = %s,
   270→                        shard_id = %s,
   271→                        last_error = %s,
   272→                        claimed_by = null,
   273→                        lease_expires_at = null,
   274→                        updated_at = now()
   275→                    where video_id = any(%s)
   276→                    """,
   277→                    (status, str(shard_id) if shard_id else None, error, video_ids),
   278→                )
   279→            else:
   280→                cur.execute(
   281→                    """
   282→                    update video_tasks
   283→                    set status = %s,
   284→                        shard_id = %s,
   285→                        last_error = %s,
   286→                        updated_at = now()
   287→                    where video_id = any(%s)
   288→                    """,
   289→                    (status, str(shard_id) if shard_id else None, error, video_ids),
   290→                )
   291→
   292→    def update_shard_status(
   293→        self,
   294→        conn,
   295→        shard_id: uuid.UUID,
   296→        status: str,
   297→        metrics: Optional[dict] = None,
   298→    ) -> None:
   299→        """Update shard_runs status."""
   300→        with conn.cursor() as cur:
   301→            cur.execute(
   302→                """
   303→                update shard_runs
   304→                set status = %s,
   305→                    metrics = coalesce(%s, metrics),
   306→                    updated_at = now()
   307→                where shard_id = %s
   308→                """,
   309→                (status, json.dumps(metrics) if metrics else None, str(shard_id)),
   310→            )
   311→
   312→    def renew_lease(
   313→        self,
   314→        conn,
   315→        shard_id: Optional[uuid.UUID],
   316→        video_ids: list[str],
   317→        language: Optional[str] = None,
   318→    ) -> None:
   319→        """Renew lease for current shard and videos."""
   320→        with conn.cursor() as cur:
   321→            if shard_id is not None:
   322→                # Renew shard lease
   323→                cur.execute(
   324→                    """
   325→                    update shard_runs
   326→                    set lease_expires_at = now() + (%s || ' minutes')::interval,
   327→                        updated_at = now()
   328→                    where shard_id = %s
   329→                    """,
   330→                    (str(self.config.lease_minutes), str(shard_id)),
   331→                )
   332→            if video_ids:
   333→                # Renew video leases
   334→                cur.execute(
   335→                    """
   336→                    update video_tasks
   337→                    set lease_expires_at = now() + (%s || ' minutes')::interval,
   338→                        updated_at = now()
   339→                    where video_id = any(%s)
   340→                    """,
   341→                    (str(self.config.lease_minutes), video_ids),
   342→                )
   343→            if language and self.config.auto_language:
   344→                cur.execute(
   345→                    """
   346→                    update language_queue
   347→                    set lease_expires_at = now() + (%s || ' minutes')::interval,
   348→                        updated_at = now()
   349→                    where language = %s
   350→                      and status = 'processing'
   351→                    """,
   352→                    (str(self.config.lease_minutes), language),
   353→                )
   354→
   355→    def ensure_active_language(self, conn) -> Optional[str]:
   356→        """Get or claim the active language when auto-language mode is enabled."""
   357→        if not self.config.auto_language:
   358→            return self.config.language or None
   359→
   360→        with conn.cursor() as cur:
   361→            cur.execute(
   362→                """
   363→                select active_language
   364→                from language_state
   365→                where id = 1
   366→                for update
   367→                """
   368→            )
   369→            row = cur.fetchone()
   370→            if row and row[0]:
   371→                return row[0]
   372→
   373→            cur.execute(
   374→                """
   375→                with picked as (
   376→                    select language
   377→                    from language_queue
   378→                    where status = 'pending'
   379→                    order by priority desc, language
   380→                    limit 1
   381→                    for update skip locked
   382→                )
   383→                update language_queue
   384→                set status = 'processing',
   385→                    claimed_by = %s,
   386→                    lease_expires_at = now() + (%s || ' minutes')::interval,
   387→                    updated_at = now()
   388→                where language in (select language from picked)
   389→                returning language
   390→                """,
   391→                (self.worker_id, str(self.config.lease_minutes)),
   392→            )
   393→            picked = cur.fetchone()
   394→            if not picked:
   395→                return None
   396→            language = picked[0]
   397→            cur.execute(
   398→                """
   399→                update language_state
   400→                set active_language = %s,
   401→                    updated_at = now()
   402→                where id = 1
   403→                """,
   404→                (language,),
   405→            )
   406→            return language
   407→
   408→    def language_has_pending(self, conn, language: str) -> bool:
   409→        """Check if a language still has any pending/active work."""
   410→        with conn.cursor() as cur:
   411→            cur.execute(
   412→                """
   413→                select count(*)
   414→                from video_tasks
   415→                where language = %s
   416→                  and status in ('pending', 'claimed', 'processing', 'uploading')
   417→                """,
   418→                (language,),
   419→            )
   420→            return cur.fetchone()[0] > 0
   421→
   422→    def mark_language_completed(self, conn, language: str) -> None:
   423→        """Mark language as completed and clear active language."""
   424→        with conn.cursor() as cur:
   425→            cur.execute(
   426→                """
   427→                update language_queue
   428→                set status = 'completed',
   429→                    completed_at = now(),
   430→                    updated_at = now()
   431→                where language = %s
   432→                """,
   433→                (language,),
   434→            )
   435→            cur.execute(
   436→                """
   437→                update language_state
   438→                set active_language = null,
   439→                    updated_at = now()
   440→                where id = 1
   441→                  and active_language = %s
   442→                """,
   443→                (language,),
   444→            )
   445→
   446→    def send_heartbeat(self, conn) -> None:
   447→        """Send heartbeat to database."""
   448→        state = self._get_state()
   449→        storage = self.storage_manager.get_stats()
   450→
   451→        with conn.cursor() as cur:
   452→            cur.execute(
   453→                """
   454→                insert into worker_heartbeats_v2 (
   455→                    worker_id, hostname, instance_type, language, status,
   456→                    shard_id, videos_assigned, videos_processed,
   457→                    segments_processed, outputs_generated, last_error, last_heartbeat
   458→                ) values (
   459→                    %s, %s, %s, %s, %s,
   460→                    %s, %s, %s,
   461→                    %s, %s, %s, now()
   462→                )
   463→                on conflict (worker_id) do update set
   464→                    status = excluded.status,
   465→                    shard_id = excluded.shard_id,
   466→                    videos_assigned = excluded.videos_assigned,
   467→                    videos_processed = excluded.videos_processed,
   468→                    segments_processed = excluded.segments_processed,
   469→                    outputs_generated = excluded.outputs_generated,
   470→                    last_error = excluded.last_error,
   471→                    last_heartbeat = now()
   472→                """,
   473→                (
   474→                    self.worker_id,
   475→                    socket.gethostname(),
   476→                    os.environ.get("INSTANCE_TYPE", "unknown"),
   477→                    self.config.language,
   478→                    state["status"],
   479→                    state["shard_id"],
   480→                    state["videos_assigned"],
   481→                    state["videos_processed"],
   482→                    state["segments_processed"],
   483→                    state["outputs_generated"],
   484→                    state["last_error"],
   485→                ),
   486→            )
   487→
   488→    def _heartbeat_loop(self) -> None:
   489→        """Background heartbeat loop."""
   490→        while not self._stop_event.is_set():
   491→            try:
   492→                with psycopg.connect(self.env_config.database_url) as conn:
   493→                    self.send_heartbeat(conn)
   494→                    conn.commit()
   495→            except Exception as e:
   496→                logger.warning(f"Heartbeat failed: {e}")
   497→
   498→            self._stop_event.wait(self.config.heartbeat_interval_sec)
   499→
   500→    def start_heartbeat(self) -> None:
   501→        """Start background heartbeat thread."""
   502→        if self._heartbeat_thread is not None:
   503→            return
   504→        self._heartbeat_thread = threading.Thread(
   505→            target=self._heartbeat_loop,
   506→            name="heartbeat",
   507→            daemon=True,
   508→        )
   509→        self._heartbeat_thread.start()
   510→        logger.info("Heartbeat thread started")
   511→
   512→    def stop_heartbeat(self) -> None:
   513→        """Stop heartbeat thread."""
   514→        self._stop_event.set()
   515→        if self._heartbeat_thread:
   516→            self._heartbeat_thread.join(timeout=5)
   517→            self._heartbeat_thread = None
   518→
   519→    def _lease_loop(
   520→        self,
   521→        shard_id: Optional[uuid.UUID],
   522→        video_ids: list[str],
   523→        language: Optional[str],
   524→    ) -> None:
   525→        """Background lease renewal loop while processing a shard."""
   526→        interval_sec = max(60, int(self.config.lease_minutes * 60 / 3))
   527→        while not self._lease_stop_event.is_set() and not self._stop_event.is_set():
   528→            try:
   529→                with psycopg.connect(self.env_config.database_url) as conn:
   530→                    self.renew_lease(conn, shard_id, video_ids, language=language)
   531→                    conn.commit()
   532→            except Exception as e:
   533→                logger.warning(f"Lease renewal failed: {e}")
   534→            self._lease_stop_event.wait(interval_sec)
   535→
   536→    def start_lease_renewal(
   537→        self,
   538→        shard_id: Optional[uuid.UUID],
   539→        video_ids: list[str],
   540→        language: Optional[str] = None,
   541→    ) -> None:
   542→        """Start background lease renewal for the active shard."""
   543→        self.stop_lease_renewal()
   544→        self._lease_stop_event.clear()
   545→        self._lease_thread = threading.Thread(
   546→            target=self._lease_loop,
   547→            args=(shard_id, list(video_ids), language),
   548→            name="lease-renewal",
   549→            daemon=True,
   550→        )
   551→        self._lease_thread.start()
   552→        logger.info("Lease renewal thread started")
   553→
   554→    def stop_lease_renewal(self) -> None:
   555→        """Stop background lease renewal."""
   556→        self._lease_stop_event.set()
   557→        if self._lease_thread:
   558→            self._lease_thread.join(timeout=5)
   559→            self._lease_thread = None
   560→
   561→    def download_videos(
   562→        self,
   563→        tasks: list[TaskRow],
   564→        download_dir: Path,
   565→    ) -> dict[str, Path]:
   566→        """Download video tars from R2."""
   567→        download_dir.mkdir(parents=True, exist_ok=True)
   568→        download_paths: dict[str, Path] = {}
   569→
   570→        transfer_config = TransferConfig(
   571→            multipart_threshold=8 * 1024 * 1024,
   572→            multipart_chunksize=64 * 1024 * 1024,
   573→            max_concurrency=max(4, self.config.download_workers),
   574→            use_threads=True,
   575→        )
   576→
   577→        for i, task in enumerate(tasks):
   578→            if self._stop_event.is_set():
   579→                break
   580→
   581→            # Check storage before download
   582→            if self.storage_manager.needs_cleanup():
   583→                logger.warning("Storage threshold exceeded, running cleanup...")
   584→                self.storage_manager.cleanup()
   585→
   586→            local_path = download_dir / f"{task.video_id}.tar"
   587→            if local_path.exists():
   588→                download_paths[task.video_id] = local_path
   589→                continue
   590→
   591→            # Resolve R2 key
   592→            resolved = self.source_uploader.resolve_key(
   593→                task.r2_tar_key,
   594→                self.env_config.r2_prefix,
   595→            )
   596→            if not resolved:
   597→                logger.warning(f"Could not resolve R2 key: {task.r2_tar_key}")
   598→                continue
   599→
   600→            # Download
   601→            try:
   602→                if self.source_uploader.download_file(resolved, local_path):
   603→                    download_paths[task.video_id] = local_path
   604→                    logger.debug(f"Downloaded {task.video_id} ({i+1}/{len(tasks)})")
   605→                else:
   606→                    logger.error(f"Failed to download {task.video_id}")
   607→            except Exception as e:
   608→                logger.error(f"Download error for {task.video_id}: {e}")
   609→
   610→            # Update progress
   611→            self._update_state(videos_processed=len(download_paths))
   612→
   613→        return download_paths
   614→
   615→    def process_shard(
   616→        self,
   617→        shard_id: uuid.UUID,
   618→        download_paths: dict[str, Path],
   619→    ) -> ProcessingStats:
   620→        """Process downloaded videos into shards."""
   621→        self.config.output_dir.mkdir(parents=True, exist_ok=True)
   622→
   623→        chunk_cfg = ChunkConfig(
   624→            enabled=self.config.chunk_enabled,
   625→            min_sec=self.config.chunk_min_sec,
   626→            max_sec=self.config.chunk_max_sec,
   627→            threshold_sec=self.config.chunk_threshold_sec,
   628→            all_buckets=self.config.chunk_all_buckets,
   629→        )
   630→        silence_cfg = SilenceConfig(
   631→            enabled=self.config.silence_enabled,
   632→            frame_sec=self.config.silence_frame_ms / 1000.0,
   633→            hop_sec=self.config.silence_hop_ms / 1000.0,
   634→            search_sec=self.config.silence_search_sec,
   635→            percentile=self.config.silence_percentile,
   636→            db_drop=self.config.silence_db_drop,
   637→        )
   638→
   639→        # Use shard_id as prefix for deterministic output naming
   640→        shard_prefix = f"shard-{str(shard_id)}"
   641→
   642→        def on_progress(processed: int, total: int):
   643→            self._update_state(videos_processed=processed)
   644→            # Check storage during processing
   645→            if processed % 10 == 0 and self.storage_manager.needs_cleanup():
   646→                logger.warning("Storage threshold during processing, running cleanup...")
   647→                self.storage_manager.cleanup()
   648→
   649→        processor = VideoProcessor(
   650→            output_dir=self.config.output_dir,
   651→            language=self.config.language,
   652→            shard_prefix=shard_prefix,
   653→            chunk_cfg=chunk_cfg,
   654→            silence_cfg=silence_cfg,
   655→            output_format=self.config.output_format,
   656→            target_shard_size_mb=self.config.target_shard_size_mb,
   657→            segment_workers=self.config.segment_workers,
   658→            seed=self.config.seed,
   659→            val_ratio=self.config.val_ratio,
   660→            on_progress=on_progress,
   661→        )
   662→
   663→        try:
   664→            stats = processor.process_videos(download_paths)
   665→            processor.close()
   666→            processor.save_stats()
   667→
   668→            # Update state with processing stats
   669→            total_outputs = sum(stats.segments_kept.values())
   670→            self._update_state(
   671→                segments_processed=stats.segments_total,
   672→                outputs_generated=total_outputs,
   673→            )
   674→
   675→            return stats
   676→        except Exception as e:
   677→            processor.close()
   678→            raise
   679→
   680→    def upload_shard(
   681→        self,
   682→        shard_id: uuid.UUID,
   683→    ) -> bool:
   684→        """Upload processed shard to R2."""
   685→        result = self.shard_uploader.upload_shard(
   686→            output_dir=self.config.output_dir,
   687→            language=self.config.language,
   688→            shard_id=str(shard_id),
   689→            audio_ext="wav" if self.config.output_format == "wav" else "flac",
   690→        )
   691→
   692→        if result.success:
   693→            logger.info(
   694→                f"Uploaded shard {shard_id}: {len(result.uploaded_keys)} files, "
   695→                f"{result.total_bytes / (1024**3):.2f}GB"
   696→            )
   697→        else:
   698→            logger.error(f"Upload failed for shard {shard_id}: {result.errors}")
   699→
   700→        return result.success
   701→
   702→    def process_one_shard(self) -> bool:
   703→        """Process a single shard from claim to upload.
   704→
   705→        Returns True if a shard was processed, False if no work available.
   706→        """
   707→        # Claim videos
   708→        self._update_state(status="claiming")
   709→        tasks: list[TaskRow] = []
   710→        claimed_video_ids: list[str] = []
   711→        active_video_ids: list[str] = []
   712→        shard_id: Optional[uuid.UUID] = None
   713→        download_dir: Optional[Path] = None
   714→        language: Optional[str] = None
   715→        with psycopg.connect(self.env_config.database_url) as conn:
   716→            attempts = 0
   717→            while True:
   718→                attempts += 1
   719→                conn.execute("begin")
   720→                language = self.ensure_active_language(conn)
   721→
   722→                if not language:
   723→                    conn.execute("commit")
   724→                    logger.info("No pending languages available")
   725→                    return False
   726→
   727→                self.config.language = language
   728→                tasks = self.claim_videos(conn, language)
   729→
   730→                if not tasks:
   731→                    if self.config.auto_language and not self.language_has_pending(conn, language):
   732→                        logger.info(f"Language {language} completed, advancing queue")
   733→                        self.mark_language_completed(conn, language)
   734→                        conn.execute("commit")
   735→                        if attempts < 2:
   736→                            continue
   737→                        return False
   738→                    conn.execute("commit")
   739→                    logger.info(f"No pending videos available for language {language}")
   740→                    return False
   741→
   742→                claimed_video_ids = [t.video_id for t in tasks]
   743→                logger.info(f"Claimed {len(tasks)} videos (downloads pending)")
   744→                conn.execute("commit")
   745→                break
   746→
   747→        try:
   748→            self.start_lease_renewal(None, claimed_video_ids, language=language)
   749→            # Download
   750→            self._update_state(status="downloading")
   751→            download_dir = self.config.work_dir / f"claim-{self.worker_id}-{int(time.time())}"
   752→            download_paths = self.download_videos(tasks, download_dir)
   753→
   754→            active_video_ids = sorted(download_paths.keys())
   755→            missing_ids = [vid for vid in claimed_video_ids if vid not in download_paths]
   756→
   757→            if missing_ids:
   758→                logger.warning(
   759→                    f"Missing downloads for {len(missing_ids)} videos; "
   760→                    "marking as failed"
   761→                )
   762→                with psycopg.connect(self.env_config.database_url) as conn:
   763→                    self.update_task_status(
   764→                        conn,
   765→                        missing_ids,
   766→                        "failed",
   767→                        error="download_failed",
   768→                        clear_claim=True,
   769→                    )
   770→                    conn.commit()
   771→
   772→            if not active_video_ids:
   773→                raise RuntimeError("No videos downloaded successfully")
   774→
   775→            shard_id = self.deterministic_shard_id(self.config.language, active_video_ids)
   776→            logger.info(f"Processing {len(active_video_ids)} videos, shard_id={shard_id}")
   777→
   778→            final_download_dir = self.config.work_dir / str(shard_id)
   779→            if download_dir != final_download_dir:
   780→                if final_download_dir.exists():
   781→                    shutil.rmtree(final_download_dir)
   782→                shutil.move(str(download_dir), str(final_download_dir))
   783→                download_dir = final_download_dir
   784→                download_paths = {
   785→                    vid: final_download_dir / path.name
   786→                    for vid, path in download_paths.items()
   787→                }
   788→
   789→            with psycopg.connect(self.env_config.database_url) as conn:
   790→                conn.execute("begin")
   791→                self.create_shard_run(
   792→                    conn,
   793→                    shard_id,
   794→                    len(active_video_ids),
   795→                    status="processing",
   796→                    language=self.config.language,
   797→                )
   798→                self.update_task_status(conn, active_video_ids, "processing", shard_id=shard_id)
   799→                conn.execute("commit")
   800→
   801→            self._update_state(
   802→                status="processing",
   803→                shard_id=str(shard_id),
   804→                videos_assigned=len(active_video_ids),
   805→                videos_processed=0,
   806→                processing_start=time.time(),
   807→            )
   808→
   809→            self.start_lease_renewal(shard_id, active_video_ids, language=language)
   810→
   811→            # Renew lease before processing
   812→            with psycopg.connect(self.env_config.database_url) as conn:
   813→                self.renew_lease(conn, shard_id, active_video_ids, language=language)
   814→                conn.commit()
   815→
   816→            # Process
   817→            self._update_state(status="processing")
   818→            stats = self.process_shard(shard_id, download_paths)
   819→
   820→            # Renew lease before upload
   821→            with psycopg.connect(self.env_config.database_url) as conn:
   822→                self.renew_lease(conn, shard_id, active_video_ids, language=language)
   823→                conn.commit()
   824→
   825→            # Upload
   826→            self._update_state(status="uploading")
   827→            with psycopg.connect(self.env_config.database_url) as conn:
   828→                self.update_shard_status(conn, shard_id, "uploading")
   829→                self.update_task_status(conn, active_video_ids, "uploading", shard_id=shard_id)
   830→                conn.commit()
   831→            if not self.upload_shard(shard_id):
   832→                raise RuntimeError("Upload failed")
   833→
   834→            # Mark completed
   835→            with psycopg.connect(self.env_config.database_url) as conn:
   836→                self.update_shard_status(conn, shard_id, "completed", metrics=stats.to_dict())
   837→                self.update_task_status(conn, active_video_ids, "completed", shard_id=shard_id)
   838→                conn.commit()
   839→
   840→            logger.info(f"Completed shard {shard_id} with {len(download_paths)} videos")
   841→
   842→            # Cleanup
   843→            self._update_state(status="cleanup")
   844→            if download_dir is not None:
   845→                self.storage_manager.cleanup_shard_data(str(shard_id), download_dir)
   846→            self.storage_manager.mark_shard_uploaded(str(shard_id))
   847→
   848→            self._update_state(status="idle", shard_id=None)
   849→            return True
   850→
   851→        except Exception as e:
   852→            logger.exception(f"Failed to process shard {shard_id}")
   853→            self._update_state(status="failed", last_error=str(e))
   854→
   855→            # Mark as failed in database
   856→            try:
   857→                with psycopg.connect(self.env_config.database_url) as conn:
   858→                    if shard_id is not None:
   859→                        self.update_shard_status(conn, shard_id, "failed")
   860→                    self.update_task_status(
   861→                        conn,
   862→                        active_video_ids,
   863→                        "failed",
   864→                        shard_id=shard_id,
   865→                        error=str(e),
   866→                        clear_claim=True,
   867→                    )
   868→                    conn.commit()
   869→            except Exception as db_err:
   870→                logger.error(f"Failed to update failure status: {db_err}")
   871→
   872→            return False
   873→        finally:
   874→            self.stop_lease_renewal()
   875→
   876→    def run(self) -> None:
   877→        """Main worker loop."""
   878→        logger.info(f"Starting worker for language={self.config.language}")
   879→
   880→        # Validate config
   881→        errors = self.env_config.validate()
   882→        if errors:
   883→            for error in errors:
   884→                logger.error(f"Config error: {error}")
   885→            raise SystemExit(1)
   886→        if not self.config.auto_language and not self.config.language:
   887→            logger.error("LANGUAGE is required unless AUTO_LANGUAGE is set")
   888→            raise SystemExit(1)
   889→
   890→        # Start background threads
   891→        self.start_heartbeat()
   892→        self.storage_manager.start_monitor()
   893→
   894→        # Set storage callbacks
   895→        def on_storage_critical(stats: StorageStats):
   896→            logger.warning(
   897→                f"Storage critical: {stats.work_dir_gb + stats.output_dir_gb:.1f}GB used, "
   898→                f"{stats.free_gb:.1f}GB free"
   899→            )
   900→            self._update_state(last_error="Storage critical")
   901→
   902→        self.storage_manager.set_callbacks(on_storage_critical=on_storage_critical)
   903→
   904→        try:
   905→            while not self._stop_event.is_set():
   906→                try:
   907→                    processed = self.process_one_shard()
   908→
   909→                    if self.config.once:
   910→                        logger.info("Single shard mode, exiting")
   911→                        break
   912→
   913→                    if not processed:
   914→                        # No work available, wait before retry
   915→                        logger.info("No work available, sleeping 30s...")
   916→                        self._stop_event.wait(30)
   917→
   918→                except Exception as e:
   919→                    logger.exception(f"Error in main loop: {e}")
   920→                    self._update_state(status="error", last_error=str(e))
   921→                    # Wait before retry
   922→                    self._stop_event.wait(60)
   923→
   924→        finally:
   925→            logger.info("Shutting down worker...")
   926→            self.stop_heartbeat()
   927→            self.storage_manager.stop_monitor()
   928→
   929→            # Final heartbeat with idle status
   930→            try:
   931→                self._update_state(status="stopped")
   932→                with psycopg.connect(self.env_config.database_url) as conn:
   933→                    self.send_heartbeat(conn)
   934→                    conn.commit()
   935→            except Exception:
   936→                pass
   937→
   938→            logger.info("Worker shutdown complete")
   939→
   940→
   941→def parse_args() -> argparse.Namespace:
   942→    """Parse command line arguments."""
   943→    parser = argparse.ArgumentParser(description="Distributed shard worker")
   944→
   945→    # Required
   946→    parser.add_argument("--language", default="", help="Language to process")
   947→    parser.add_argument(
   948→        "--auto-language",
   949→        action="store_true",
   950→        help="Enable language queue (ignore --language)",
   951→    )
   952→
   953→    # Worker identity
   954→    parser.add_argument("--worker-id", default=None, help="Worker ID (auto-generated if not set)")
   955→
   956→    # Processing settings
   957→    parser.add_argument("--target-videos", type=int, default=100, help="Videos per shard")
   958→    parser.add_argument("--lease-minutes", type=int, default=60, help="Lease duration")
   959→
   960→    # Directories
   961→    parser.add_argument("--work-dir", type=Path, default=Path("/tmp/worker"))
   962→    parser.add_argument("--output-dir", type=Path, default=Path("/tmp/worker_output"))
   963→
   964→    # Storage
   965→    parser.add_argument("--max-disk-gb", type=float, default=150.0, help="Max disk usage GB")
   966→    parser.add_argument("--cleanup-threshold-gb", type=float, default=100.0)
   967→
   968→    # Parallelism
   969→    parser.add_argument("--download-workers", type=int, default=8)
   970→    parser.add_argument("--segment-workers", type=int, default=8)
   971→    parser.add_argument("--upload-workers", type=int, default=4)
   972→
   973→    # Output
   974→    parser.add_argument("--output-format", choices=["flac", "wav"], default="flac")
   975→    parser.add_argument(
   976→        "--shard-size-mb",
   977→        type=int,
   978→        default=0,
   979→        help="Target shard size MB (0 disables size-based splitting)",
   980→    )
   981→
   982→    # Run mode
   983→    parser.add_argument("--once", action="store_true", help="Process single shard and exit")
   984→    parser.add_argument("--dry-run", action="store_true", help="Don't actually process")
   985→
   986→    # Config file
   987→    parser.add_argument("--env-file", type=Path, default=Path(".env"))
   988→
   989→    return parser.parse_args()
   990→
   991→
   992→def main():
   993→    args = parse_args()
   994→
   995→    # Load environment config
   996→    if args.env_file.exists():
   997→        env_config = EnvConfig.from_file(args.env_file)
   998→    else:
   999→        env_config = EnvConfig.from_env_vars()
  1000→
  1001→    # Create worker config from args
  1002→    worker_config = WorkerConfig(
  1003→        worker_id=args.worker_id or "",
  1004→        language=args.language,
  1005→        auto_language=args.auto_language,
  1006→        target_videos=args.target_videos,
  1007→        lease_minutes=args.lease_minutes,
  1008→        work_dir=args.work_dir,
  1009→        output_dir=args.output_dir,
  1010→        max_disk_usage_gb=args.max_disk_gb,
  1011→        cleanup_threshold_gb=args.cleanup_threshold_gb,
  1012→        download_workers=args.download_workers,
  1013→        segment_workers=args.segment_workers,
  1014→        upload_workers=args.upload_workers,
  1015→        output_format=args.output_format,
  1016→        target_shard_size_mb=args.shard_size_mb,
  1017→        once=args.once,
  1018→        dry_run=args.dry_run,
  1019→    )
  1020→
  1021→    worker = ShardWorker(worker_config, env_config)
  1022→    worker.run()
  1023→
  1024→
  1025→if __name__ == "__main__":
  1026→    main()
  1027→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

#!/usr/bin/env python3
"""Distributed shard worker for horizontal scaling.

This worker:
1. Claims a batch of videos from Supabase (with row-level locking)
2. Downloads video tars from R2
3. Processes segments to generate shards
4. Uploads shards to R2
5. Updates status in Supabase
6. Cleans up local storage
7. Reports heartbeat and metrics

Key safety features:
- Deterministic shard_id (same videos = same shard = idempotent upload)
- FOR UPDATE SKIP LOCKED prevents duplicate claims
- Lease expiry allows crash recovery
- Storage monitoring prevents disk exhaustion
"""

from __future__ import annotations

import argparse
import hashlib
import json
import logging
import os
import signal
import shutil
import socket
import sys
import threading
import time
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional

import psycopg
from boto3.s3.transfer import TransferConfig

# Setup logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger("worker")

if __package__ in (None, ""):
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

# Import worker modules
from workers.config import WorkerConfig, EnvConfig
from workers.storage_manager import StorageManager, StorageStats
from workers.r2_uploader import R2Uploader
from workers.worker_processing import (
    VideoProcessor,
    ChunkConfig,
    SilenceConfig,
    ProcessingStats,
)

@dataclass(frozen=True)
class TaskRow:
    """A video task from the database."""
    video_id: str
    r2_tar_key: str
    language: str

@dataclass
class WorkerState:
    """Current worker state for heartbeat reporting."""
    status: str = "idle"
    shard_id: Optional[str] = None
    videos_assigned: int = 0
    videos_processed: int = 0
    segments_processed: int = 0
    outputs_generated: int = 0
    last_error: Optional[str] = None
    processing_start: Optional[float] = None

class ShardWorker:
    """Main worker class that orchestrates the processing pipeline."""

    def __init__(
        self,
        config: WorkerConfig,
        env_config: EnvConfig,
    ):
        self.config = config
        self.env_config = env_config

        # Generate worker ID if not provided
        self.worker_id = config.worker_id or f"{socket.gethostname()}-{os.getpid()}"
        logger.info(f"Worker ID: {self.worker_id}")

        # Initialize R2 clients
        self.source_uploader = R2Uploader(
            endpoint_url=env_config.r2_endpoint_url,
            access_key_id=env_config.r2_access_key_id,
            secret_access_key=env_config.r2_secret_access_key,
            bucket=env_config.r2_cleaned_bucket,
            max_retries=config.max_retries,
            retry_delay_sec=config.retry_delay_sec,
            max_concurrency=config.download_workers,
        )

        self.shard_uploader = R2Uploader(
            endpoint_url=env_config.r2_endpoint_url,
            access_key_id=env_config.r2_access_key_id,
            secret_access_key=env_config.r2_secret_access_key,
            bucket=env_config.r2_bucket_shards,
            max_retries=config.max_retries,
            retry_delay_sec=config.retry_delay_sec,
            max_concurrency=config.upload_workers,
        )

        # Initialize storage manager
        self.storage_manager = StorageManager(
            work_dir=config.work_dir,
            output_dir=config.output_dir,
            max_usage_gb=config.max_disk_usage_gb,
            cleanup_threshold_gb=config.cleanup_threshold_gb,
            check_interval_sec=config.cleanup_check_interval_sec,
        )

        # Worker state
        self.state = WorkerState()
        self._state_lock = threading.Lock()
        self._stop_event = threading.Event()
        self._heartbeat_thread: Optional[threading.Thread] = None
        self._lease_thread: Optional[threading.Thread] = None
        self._lease_stop_event = threading.Event()

        # Setup signal handlers
        signal.signal(signal.SIGTERM, self._handle_signal)
        signal.signal(signal.SIGINT, self._handle_signal)

    def _handle_signal(self, signum, frame):
        """Handle shutdown signals gracefully."""
        logger.info(f"Received signal {signum}, initiating graceful shutdown...")
        self._stop_event.set()

    def _update_state(self, **kwargs) -> None:
        """Thread-safe state update."""
        with self._state_lock:
            for key, value in kwargs.items():
                if hasattr(self.state, key):
                    setattr(self.state, key, value)

    def _get_state(self) -> dict:
        """Get current state as dict."""
        with self._state_lock:
            return {
                "status": self.state.status,
                "shard_id": self.state.shard_id,
                "videos_assigned": self.state.videos_assigned,
                "videos_processed": self.state.videos_processed,
                "segments_processed": self.state.segments_processed,
                "outputs_generated": self.state.outputs_generated,
                "last_error": self.state.last_error,
            }

    def deterministic_shard_id(self, language: str, video_ids: list[str]) -> uuid.UUID:
        """Generate deterministic shard ID from language + sorted video IDs.

        This ensures that re-processing the same videos produces the same shard_id,
        making uploads idempotent (overwrites same R2 key instead of creating duplicates).
        """
        joined = f"{language}:" + ",".join(sorted(video_ids))
        digest = hashlib.sha1(joined.encode("utf-8")).hexdigest()
        return uuid.uuid5(uuid.NAMESPACE_URL, digest)

    def claim_videos(self, conn, language: str) -> list[TaskRow]:
        """Claim a batch of pending videos with row-level locking.

        Uses FOR UPDATE SKIP LOCKED to prevent duplicate claims across workers.
        """
        with conn.cursor() as cur:
            cur.execute(
                """
                with picked as (
                    select video_id
                    from video_tasks
                    where language = %s
                      and status = 'pending'
                    order by video_id
                    limit %s
                    for update skip locked
                )
                update video_tasks
                set status = 'claimed',
                    claimed_by = %s,
                    lease_expires_at = now() + (%s || ' minutes')::interval,
                    attempts = attempts + 1,
                    updated_at = now()
                where video_id in (select video_id from picked)
                returning video_id, r2_tar_key, language
                """,
                (
                    language,
                    self.config.target_videos,
                    self.worker_id,
                    str(self.config.lease_minutes),
                ),
            )
            rows = cur.fetchall()
        return [TaskRow(*row) for row in rows]

    def create_shard_run(
        self,
        conn,
        shard_id: uuid.UUID,
        assigned_count: int,
        status: str = "claimed",
        language: Optional[str] = None,
    ) -> None:
        """Create shard_runs entry for tracking."""
        run_language = language or self.config.language
        with conn.cursor() as cur:
            cur.execute(
                """
                insert into shard_runs (
                    shard_id, language, target_video_count, assigned_video_count,
                    status, claimed_by, lease_expires_at, shard_prefix, output_format
                ) values (
                    %s, %s, %s, %s,
                    %s, %s, now() + (%s || ' minutes')::interval, %s, %s
                )
                on conflict (shard_id) do update set
                    status = excluded.status,
                    claimed_by = excluded.claimed_by,
                    lease_expires_at = excluded.lease_expires_at,
                    updated_at = now()
                """,
                (
                    str(shard_id),
                    run_language,
                    self.config.target_videos,
                    assigned_count,
                    status,
                    self.worker_id,
                    str(self.config.lease_minutes),
                    f"{run_language}/shard-{shard_id}",
                    self.config.output_format,
                ),
            )

    def update_task_status(
        self,
        conn,
        video_ids: list[str],
        status: str,
        shard_id: Optional[uuid.UUID] = None,
        error: Optional[str] = None,
        clear_claim: bool = False,
    ) -> None:
        """Update status for video tasks."""
        if not video_ids:
            return
        with conn.cursor() as cur:
            if clear_claim:
                cur.execute(
                    """
                    update video_tasks
                    set status = %s,
                        shard_id = %s,
                        last_error = %s,
                        claimed_by = null,
                        lease_expires_at = null,
                        updated_at = now()
                    where video_id = any(%s)
                    """,
                    (status, str(shard_id) if shard_id else None, error, video_ids),
                )
            else:
                cur.execute(
                    """
                    update video_tasks
                    set status = %s,
                        shard_id = %s,
                        last_error = %s,
                        updated_at = now()
                    where video_id = any(%s)
                    """,
                    (status, str(shard_id) if shard_id else None, error, video_ids),
                )

    def update_shard_status(
        self,
        conn,
        shard_id: uuid.UUID,
        status: str,
        metrics: Optional[dict] = None,
    ) -> None:
        """Update shard_runs status."""
        with conn.cursor() as cur:
            cur.execute(
                """
                update shard_runs
                set status = %s,
                    metrics = coalesce(%s, metrics),
                    updated_at = now()
                where shard_id = %s
                """,
                (status, json.dumps(metrics) if metrics else None, str(shard_id)),
            )

    def renew_lease(
        self,
        conn,
        shard_id: Optional[uuid.UUID],
        video_ids: list[str],
        language: Optional[str] = None,
    ) -> None:
        """Renew lease for current shard and videos."""
        with conn.cursor() as cur:
            if shard_id is not None:
                # Renew shard lease
                cur.execute(
                    """
                    update shard_runs
                    set lease_expires_at = now() + (%s || ' minutes')::interval,
                        updated_at = now()
                    where shard_id = %s
                    """,
                    (str(self.config.lease_minutes), str(shard_id)),
                )
            if video_ids:
                # Renew video leases
                cur.execute(
                    """
                    update video_tasks
                    set lease_expires_at = now() + (%s || ' minutes')::interval,
                        updated_at = now()
                    where video_id = any(%s)
                    """,
                    (str(self.config.lease_minutes), video_ids),
                )
            if language and self.config.auto_language:
                cur.execute(
                    """
                    update language_queue
                    set lease_expires_at = now() + (%s || ' minutes')::interval,
                        updated_at = now()
                    where language = %s
                      and status = 'processing'
                    """,
                    (str(self.config.lease_minutes), language),
                )

    def ensure_active_language(self, conn) -> Optional[str]:
        """Get or claim the active language when auto-language mode is enabled."""
        if not self.config.auto_language:
            return self.config.language or None

        with conn.cursor() as cur:
            cur.execute(
                """
                select active_language
                from language_state
                where id = 1
                for update
                """
            )
            row = cur.fetchone()
            if row and row[0]:
                return row[0]

            cur.execute(
                """
                with picked as (
                    select language
                    from language_queue
                    where status = 'pending'
                    order by priority desc, language
                    limit 1
                    for update skip locked
                )
                update language_queue
                set status = 'processing',
                    claimed_by = %s,
                    lease_expires_at = now() + (%s || ' minutes')::interval,
                    updated_at = now()
                where language in (select language from picked)
                returning language
                """,
                (self.worker_id, str(self.config.lease_minutes)),
            )
            picked = cur.fetchone()
            if not picked:
                return None
            language = picked[0]
            cur.execute(
                """
                update language_state
                set active_language = %s,
                    updated_at = now()
                where id = 1
                """,
                (language,),
            )
            return language

    def language_has_pending(self, conn, language: str) -> bool:
        """Check if a language still has any pending/active work."""
        with conn.cursor() as cur:
            cur.execute(
                """
                select count(*)
                from video_tasks
                where language = %s
                  and status in ('pending', 'claimed', 'processing', 'uploading')
                """,
                (language,),
            )
            return cur.fetchone()[0] > 0

    def mark_language_completed(self, conn, language: str) -> None:
        """Mark language as completed and clear active language."""
        with conn.cursor() as cur:
            cur.execute(
                """
                update language_queue
                set status = 'completed',
                    completed_at = now(),
                    updated_at = now()
                where language = %s
                """,
                (language,),
            )
            cur.execute(
                """
                update language_state
                set active_language = null,
                    updated_at = now()
                where id = 1
                  and active_language = %s
                """,
                (language,),
            )

    def send_heartbeat(self, conn) -> None:
        """Send heartbeat to database."""
        state = self._get_state()
        storage = self.storage_manager.get_stats()

        with conn.cursor() as cur:
            cur.execute(
                """
                insert into worker_heartbeats_v2 (
                    worker_id, hostname, instance_type, language, status,
                    shard_id, videos_assigned, videos_processed,
                    segments_processed, outputs_generated, last_error, last_heartbeat
                ) values (
                    %s, %s, %s, %s, %s,
                    %s, %s, %s,
                    %s, %s, %s, now()
                )
                on conflict (worker_id) do update set
                    status = excluded.status,
                    shard_id = excluded.shard_id,
                    videos_assigned = excluded.videos_assigned,
                    videos_processed = excluded.videos_processed,
                    segments_processed = excluded.segments_processed,
                    outputs_generated = excluded.outputs_generated,
                    last_error = excluded.last_error,
                    last_heartbeat = now()
                """,
                (
                    self.worker_id,
                    socket.gethostname(),
                    os.environ.get("INSTANCE_TYPE", "unknown"),
                    self.config.language,
                    state["status"],
                    state["shard_id"],
                    state["videos_assigned"],
                    state["videos_processed"],
                    state["segments_processed"],
                    state["outputs_generated"],
                    state["last_error"],
                ),
            )

    def _heartbeat_loop(self) -> None:
        """Background heartbeat loop."""
        while not self._stop_event.is_set():
            try:
                with psycopg.connect(self.env_config.database_url) as conn:
                    self.send_heartbeat(conn)
                    conn.commit()
            except Exception as e:
                logger.warning(f"Heartbeat failed: {e}")

            self._stop_event.wait(self.config.heartbeat_interval_sec)

    def start_heartbeat(self) -> None:
        """Start background heartbeat thread."""
        if self._heartbeat_thread is not None:
            return
        self._heartbeat_thread = threading.Thread(
            target=self._heartbeat_loop,
            name="heartbeat",
            daemon=True,
        )
        self._heartbeat_thread.start()
        logger.info("Heartbeat thread started")

    def stop_heartbeat(self) -> None:
        """Stop heartbeat thread."""
        self._stop_event.set()
        if self._heartbeat_thread:
            self._heartbeat_thread.join(timeout=5)
            self._heartbeat_thread = None

    def _lease_loop(
        self,
        shard_id: Optional[uuid.UUID],
        video_ids: list[str],
        language: Optional[str],
    ) -> None:
        """Background lease renewal loop while processing a shard."""
        interval_sec = max(60, int(self.config.lease_minutes * 60 / 3))
        while not self._lease_stop_event.is_set() and not self._stop_event.is_set():
            try:
                with psycopg.connect(self.env_config.database_url) as conn:
                    self.renew_lease(conn, shard_id, video_ids, language=language)
                    conn.commit()
            except Exception as e:
                logger.warning(f"Lease renewal failed: {e}")
            self._lease_stop_event.wait(interval_sec)

    def start_lease_renewal(
        self,
        shard_id: Optional[uuid.UUID],
        video_ids: list[str],
        language: Optional[str] = None,
    ) -> None:
        """Start background lease renewal for the active shard."""
        self.stop_lease_renewal()
        self._lease_stop_event.clear()
        self._lease_thread = threading.Thread(
            target=self._lease_loop,
            args=(shard_id, list(video_ids), language),
            name="lease-renewal",
            daemon=True,
        )
        self._lease_thread.start()
        logger.info("Lease renewal thread started")

    def stop_lease_renewal(self) -> None:
        """Stop background lease renewal."""
        self._lease_stop_event.set()
        if self._lease_thread:
            self._lease_thread.join(timeout=5)
            self._lease_thread = None

    def download_videos(
        self,
        tasks: list[TaskRow],
        download_dir: Path,
    ) -> dict[str, Path]:
        """Download video tars from R2."""
        download_dir.mkdir(parents=True, exist_ok=True)
        download_paths: dict[str, Path] = {}

        transfer_config = TransferConfig(
            multipart_threshold=8 * 1024 * 1024,
            multipart_chunksize=64 * 1024 * 1024,
            max_concurrency=max(4, self.config.download_workers),
            use_threads=True,
        )

        for i, task in enumerate(tasks):
            if self._stop_event.is_set():
                break

            # Check storage before download
            if self.storage_manager.needs_cleanup():
                logger.warning("Storage threshold exceeded, running cleanup...")
                self.storage_manager.cleanup()

            local_path = download_dir / f"{task.video_id}.tar"
            if local_path.exists():
                download_paths[task.video_id] = local_path
                continue

            # Resolve R2 key
            resolved = self.source_uploader.resolve_key(
                task.r2_tar_key,
                self.env_config.r2_prefix,
            )
            if not resolved:
                logger.warning(f"Could not resolve R2 key: {task.r2_tar_key}")
                continue

            # Download
            try:
                if self.source_uploader.download_file(resolved, local_path):
                    download_paths[task.video_id] = local_path
                    logger.debug(f"Downloaded {task.video_id} ({i+1}/{len(tasks)})")
                else:
                    logger.error(f"Failed to download {task.video_id}")
            except Exception as e:
                logger.error(f"Download error for {task.video_id}: {e}")

            # Update progress
            self._update_state(videos_processed=len(download_paths))

        return download_paths

    def process_shard(
        self,
        shard_id: uuid.UUID,
        download_paths: dict[str, Path],
    ) -> ProcessingStats:
        """Process downloaded videos into shards."""
        self.config.output_dir.mkdir(parents=True, exist_ok=True)

        chunk_cfg = ChunkConfig(
            enabled=self.config.chunk_enabled,
            min_sec=self.config.chunk_min_sec,
            max_sec=self.config.chunk_max_sec,
            threshold_sec=self.config.chunk_threshold_sec,
            all_buckets=self.config.chunk_all_buckets,
        )
        silence_cfg = SilenceConfig(
            enabled=self.config.silence_enabled,
            frame_sec=self.config.silence_frame_ms / 1000.0,
            hop_sec=self.config.silence_hop_ms / 1000.0,
            search_sec=self.config.silence_search_sec,
            percentile=self.config.silence_percentile,
            db_drop=self.config.silence_db_drop,
        )

        # Use shard_id as prefix for deterministic output naming
        shard_prefix = f"shard-{str(shard_id)}"

        def on_progress(processed: int, total: int):
            self._update_state(videos_processed=processed)
            # Check storage during processing
            if processed % 10 == 0 and self.storage_manager.needs_cleanup():
                logger.warning("Storage threshold during processing, running cleanup...")
                self.storage_manager.cleanup()

        processor = VideoProcessor(
            output_dir=self.config.output_dir,
            language=self.config.language,
            shard_prefix=shard_prefix,
            chunk_cfg=chunk_cfg,
            silence_cfg=silence_cfg,
            output_format=self.config.output_format,
            target_shard_size_mb=self.config.target_shard_size_mb,
            segment_workers=self.config.segment_workers,
            seed=self.config.seed,
            val_ratio=self.config.val_ratio,
            on_progress=on_progress,
        )

        try:
            stats = processor.process_videos(download_paths)
            processor.close()
            processor.save_stats()

            # Update state with processing stats
            total_outputs = sum(stats.segments_kept.values())
            self._update_state(
                segments_processed=stats.segments_total,
                outputs_generated=total_outputs,
            )

            return stats
        except Exception as e:
            processor.close()
            raise

    def upload_shard(
        self,
        shard_id: uuid.UUID,
    ) -> bool:
        """Upload processed shard to R2."""
        result = self.shard_uploader.upload_shard(
            output_dir=self.config.output_dir,
            language=self.config.language,
            shard_id=str(shard_id),
            audio_ext="wav" if self.config.output_format == "wav" else "flac",
        )

        if result.success:
            logger.info(
                f"Uploaded shard {shard_id}: {len(result.uploaded_keys)} files, "
                f"{result.total_bytes / (1024**3):.2f}GB"
            )
        else:
            logger.error(f"Upload failed for shard {shard_id}: {result.errors}")

        return result.success

    def process_one_shard(self) -> bool:
        """Process a single shard from claim to upload.

        Returns True if a shard was processed, False if no work available.
        """
        # Claim videos
        self._update_state(status="claiming")
        tasks: list[TaskRow] = []
        claimed_video_ids: list[str] = []
        active_video_ids: list[str] = []
        shard_id: Optional[uuid.UUID] = None
        download_dir: Optional[Path] = None
        language: Optional[str] = None
        with psycopg.connect(self.env_config.database_url) as conn:
            attempts = 0
            while True:
                attempts += 1
                conn.execute("begin")
                language = self.ensure_active_language(conn)

                if not language:
                    conn.execute("commit")
                    logger.info("No pending languages available")
                    return False

                self.config.language = language
                tasks = self.claim_videos(conn, language)

                if not tasks:
                    if self.config.auto_language and not self.language_has_pending(conn, language):
                        logger.info(f"Language {language} completed, advancing queue")
                        self.mark_language_completed(conn, language)
                        conn.execute("commit")
                        if attempts < 2:
                            continue
                        return False
                    conn.execute("commit")
                    logger.info(f"No pending videos available for language {language}")
                    return False

                claimed_video_ids = [t.video_id for t in tasks]
                logger.info(f"Claimed {len(tasks)} videos (downloads pending)")
                conn.execute("commit")
                break

        try:
            self.start_lease_renewal(None, claimed_video_ids, language=language)
            # Download
            self._update_state(status="downloading")
            download_dir = self.config.work_dir / f"claim-{self.worker_id}-{int(time.time())}"
            download_paths = self.download_videos(tasks, download_dir)

            active_video_ids = sorted(download_paths.keys())
            missing_ids = [vid for vid in claimed_video_ids if vid not in download_paths]

            if missing_ids:
                logger.warning(
                    f"Missing downloads for {len(missing_ids)} videos; "
                    "marking as failed"
                )
                with psycopg.connect(self.env_config.database_url) as conn:
                    self.update_task_status(
                        conn,
                        missing_ids,
                        "failed",
                        error="download_failed",
                        clear_claim=True,
                    )
                    conn.commit()

            if not active_video_ids:
                raise RuntimeError("No videos downloaded successfully")

            shard_id = self.deterministic_shard_id(self.config.language, active_video_ids)
            logger.info(f"Processing {len(active_video_ids)} videos, shard_id={shard_id}")

            final_download_dir = self.config.work_dir / str(shard_id)
            if download_dir != final_download_dir:
                if final_download_dir.exists():
                    shutil.rmtree(final_download_dir)
                shutil.move(str(download_dir), str(final_download_dir))
                download_dir = final_download_dir
                download_paths = {
                    vid: final_download_dir / path.name
                    for vid, path in download_paths.items()
                }

            with psycopg.connect(self.env_config.database_url) as conn:
                conn.execute("begin")
                self.create_shard_run(
                    conn,
                    shard_id,
                    len(active_video_ids),
                    status="processing",
                    language=self.config.language,
                )
                self.update_task_status(conn, active_video_ids, "processing", shard_id=shard_id)
                conn.execute("commit")

            self._update_state(
                status="processing",
                shard_id=str(shard_id),
                videos_assigned=len(active_video_ids),
                videos_processed=0,
                processing_start=time.time(),
            )

            self.start_lease_renewal(shard_id, active_video_ids, language=language)

            # Renew lease before processing
            with psycopg.connect(self.env_config.database_url) as conn:
                self.renew_lease(conn, shard_id, active_video_ids, language=language)
                conn.commit()

            # Process
            self._update_state(status="processing")
            stats = self.process_shard(shard_id, download_paths)

            # Renew lease before upload
            with psycopg.connect(self.env_config.database_url) as conn:
                self.renew_lease(conn, shard_id, active_video_ids, language=language)
                conn.commit()

            # Upload
            self._update_state(status="uploading")
            with psycopg.connect(self.env_config.database_url) as conn:
                self.update_shard_status(conn, shard_id, "uploading")
                self.update_task_status(conn, active_video_ids, "uploading", shard_id=shard_id)
                conn.commit()
            if not self.upload_shard(shard_id):
                raise RuntimeError("Upload failed")

            # Mark completed
            with psycopg.connect(self.env_config.database_url) as conn:
                self.update_shard_status(conn, shard_id, "completed", metrics=stats.to_dict())
                self.update_task_status(conn, active_video_ids, "completed", shard_id=shard_id)
                conn.commit()

            logger.info(f"Completed shard {shard_id} with {len(download_paths)} videos")

            # Cleanup
            self._update_state(status="cleanup")
            if download_dir is not None:
                self.storage_manager.cleanup_shard_data(str(shard_id), download_dir)
            self.storage_manager.mark_shard_uploaded(str(shard_id))

            self._update_state(status="idle", shard_id=None)
            return True

        except Exception as e:
            logger.exception(f"Failed to process shard {shard_id}")
            self._update_state(status="failed", last_error=str(e))

            # Mark as failed in database
            try:
                with psycopg.connect(self.env_config.database_url) as conn:
                    if shard_id is not None:
                        self.update_shard_status(conn, shard_id, "failed")
                    self.update_task_status(
                        conn,
                        active_video_ids,
                        "failed",
                        shard_id=shard_id,
                        error=str(e),
                        clear_claim=True,
                    )
                    conn.commit()
            except Exception as db_err:
                logger.error(f"Failed to update failure status: {db_err}")

            return False
        finally:
            self.stop_lease_renewal()

    def run(self) -> None:
        """Main worker loop."""
        logger.info(f"Starting worker for language={self.config.language}")

        # Validate config
        errors = self.env_config.validate()
        if errors:
            for error in errors:
                logger.error(f"Config error: {error}")
            raise SystemExit(1)
        if not self.config.auto_language and not self.config.language:
            logger.error("LANGUAGE is required unless AUTO_LANGUAGE is set")
            raise SystemExit(1)

        # Start background threads
        self.start_heartbeat()
        self.storage_manager.start_monitor()

        # Set storage callbacks
        def on_storage_critical(stats: StorageStats):
            logger.warning(
                f"Storage critical: {stats.work_dir_gb + stats.output_dir_gb:.1f}GB used, "
                f"{stats.free_gb:.1f}GB free"
            )
            self._update_state(last_error="Storage critical")

        self.storage_manager.set_callbacks(on_storage_critical=on_storage_critical)

        try:
            while not self._stop_event.is_set():
                try:
                    processed = self.process_one_shard()

                    if self.config.once:
                        logger.info("Single shard mode, exiting")
                        break

                    if not processed:
                        # No work available, wait before retry
                        logger.info("No work available, sleeping 30s...")
                        self._stop_event.wait(30)

                except Exception as e:
                    logger.exception(f"Error in main loop: {e}")
                    self._update_state(status="error", last_error=str(e))
                    # Wait before retry
                    self._stop_event.wait(60)

        finally:
            logger.info("Shutting down worker...")
            self.stop_heartbeat()
            self.storage_manager.stop_monitor()

            # Final heartbeat with idle status
            try:
                self._update_state(status="stopped")
                with psycopg.connect(self.env_config.database_url) as conn:
                    self.send_heartbeat(conn)
                    conn.commit()
            except Exception:
                pass

            logger.info("Worker shutdown complete")

def parse_args() -> argparse.Namespace:
    """Parse command line arguments."""
    parser = argparse.ArgumentParser(description="Distributed shard worker")

    # Required
    parser.add_argument("--language", default="", help="Language to process")
    parser.add_argument(
        "--auto-language",
        action="store_true",
        help="Enable language queue (ignore --language)",
    )

    # Worker identity
    parser.add_argument("--worker-id", default=None, help="Worker ID (auto-generated if not set)")

    # Processing settings
    parser.add_argument("--target-videos", type=int, default=100, help="Videos per shard")
    parser.add_argument("--lease-minutes", type=int, default=60, help="Lease duration")

    # Directories
    parser.add_argument("--work-dir", type=Path, default=Path("/tmp/worker"))
    parser.add_argument("--output-dir", type=Path, default=Path("/tmp/worker_output"))

    # Storage
    parser.add_argument("--max-disk-gb", type=float, default=150.0, help="Max disk usage GB")
    parser.add_argument("--cleanup-threshold-gb", type=float, default=100.0)

    # Parallelism
    parser.add_argument("--download-workers", type=int, default=8)
    parser.add_argument("--segment-workers", type=int, default=8)
    parser.add_argument("--upload-workers", type=int, default=4)

    # Output
    parser.add_argument("--output-format", choices=["flac", "wav"], default="flac")
    parser.add_argument(
        "--shard-size-mb",
        type=int,
        default=0,
        help="Target shard size MB (0 disables size-based splitting)",
    )

    # Run mode
    parser.add_argument("--once", action="store_true", help="Process single shard and exit")
    parser.add_argument("--dry-run", action="store_true", help="Don't actually process")

    # Config file
    parser.add_argument("--env-file", type=Path, default=Path(".env"))

    return parser.parse_args()

def main():
    args = parse_args()

    # Load environment config
    if args.env_file.exists():
        env_config = EnvConfig.from_file(args.env_file)
    else:
        env_config = EnvConfig.from_env_vars()

    # Create worker config from args
    worker_config = WorkerConfig(
        worker_id=args.worker_id or "",
        language=args.language,
        auto_language=args.auto_language,
        target_videos=args.target_videos,
        lease_minutes=args.lease_minutes,
        work_dir=args.work_dir,
        output_dir=args.output_dir,
        max_disk_usage_gb=args.max_disk_gb,
        cleanup_threshold_gb=args.cleanup_threshold_gb,
        download_workers=args.download_workers,
        segment_workers=args.segment_workers,
        upload_workers=args.upload_workers,
        output_format=args.output_format,
        target_shard_size_mb=args.shard_size_mb,
        once=args.once,
        dry_run=args.dry_run,
    )

    worker = ShardWorker(worker_config, env_config)
    worker.run()

if __name__ == "__main__":
    main()


Let me search for more Supabase-related files and table schemas:


Bash

find /home/ubuntu/maya3_prepare -name "*.sql" -o -name "*schema*" -o -name "*seed*" | head -20

Find SQL schema and seed files


     1→#!/usr/bin/env python3
     2→"""Seed language_queue from the existing videos table or a provided list."""
     3→
     4→from __future__ import annotations
     5→
     6→import argparse
     7→from pathlib import Path
     8→
     9→import psycopg
    10→
    11→from workers.config import EnvConfig
    12→
    13→
    14→def parse_args() -> argparse.Namespace:
    15→    parser = argparse.ArgumentParser(description="Seed language_queue table.")
    16→    parser.add_argument(
    17→        "--languages",
    18→        help="Comma-separated language list (overrides auto-discovery)",
    19→    )
    20→    parser.add_argument(
    21→        "--min-count",
    22→        type=int,
    23→        default=1,
    24→        help="Minimum video count to include when auto-discovering languages",
    25→    )
    26→    parser.add_argument("--dry-run", action="store_true")
    27→    return parser.parse_args()
    28→
    29→
    30→def main() -> None:
    31→    args = parse_args()
    32→    env = EnvConfig.from_file(Path(".env"))
    33→    if not env.database_url:
    34→        raise SystemExit("DATABASE_URL missing in .env")
    35→
    36→    with psycopg.connect(env.database_url) as conn:
    37→        with conn.cursor() as cur:
    38→            if args.languages:
    39→                languages = [lang.strip().lower() for lang in args.languages.split(",") if lang.strip()]
    40→                rows = [(lang, 0) for lang in languages]
    41→            else:
    42→                cur.execute(
    43→                    """
    44→                    select lower(language), count(*)
    45→                    from videos
    46→                    where r2_tar_key is not null
    47→                      and status = 'COMPLETED'
    48→                    group by lower(language)
    49→                    having count(*) >= %s
    50→                    order by count(*) desc
    51→                    """,
    52→                    (args.min_count,),
    53→                )
    54→                rows = cur.fetchall()
    55→
    56→            if args.dry_run:
    57→                print(f"Would insert {len(rows)} languages into language_queue")
    58→                return
    59→
    60→            for language, count in rows:
    61→                cur.execute(
    62→                    """
    63→                    insert into language_queue (language, priority)
    64→                    values (%s, %s)
    65→                    on conflict (language) do nothing
    66→                    """,
    67→                    (language, int(count)),
    68→                )
    69→        conn.commit()
    70→
    71→    print(f"Inserted {len(rows)} languages into language_queue")
    72→
    73→
    74→if __name__ == "__main__":
    75→    main()
    76→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

#!/usr/bin/env python3
"""Seed language_queue from the existing videos table or a provided list."""

from __future__ import annotations

import argparse
from pathlib import Path

import psycopg

from workers.config import EnvConfig

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Seed language_queue table.")
    parser.add_argument(
        "--languages",
        help="Comma-separated language list (overrides auto-discovery)",
    )
    parser.add_argument(
        "--min-count",
        type=int,
        default=1,
        help="Minimum video count to include when auto-discovering languages",
    )
    parser.add_argument("--dry-run", action="store_true")
    return parser.parse_args()

def main() -> None:
    args = parse_args()
    env = EnvConfig.from_file(Path(".env"))
    if not env.database_url:
        raise SystemExit("DATABASE_URL missing in .env")

    with psycopg.connect(env.database_url) as conn:
        with conn.cursor() as cur:
            if args.languages:
                languages = [lang.strip().lower() for lang in args.languages.split(",") if lang.strip()]
                rows = [(lang, 0) for lang in languages]
            else:
                cur.execute(
                    """
                    select lower(language), count(*)
                    from videos
                    where r2_tar_key is not null
                      and status = 'COMPLETED'
                    group by lower(language)
                    having count(*) >= %s
                    order by count(*) desc
                    """,
                    (args.min_count,),
                )
                rows = cur.fetchall()

            if args.dry_run:
                print(f"Would insert {len(rows)} languages into language_queue")
                return

            for language, count in rows:
                cur.execute(
                    """
                    insert into language_queue (language, priority)
                    values (%s, %s)
                    on conflict (language) do nothing
                    """,
                    (language, int(count)),
                )
        conn.commit()

    print(f"Inserted {len(rows)} languages into language_queue")

if __name__ == "__main__":
    main()


Found 4 files
workers/seed_video_tasks.py
workers/release_stale_claims.py
workers/seed_language_queue.py
workers/worker.py


Perfect! Now let me read the schema file:


     1→-- Supabase schema for horizontal workers
     2→
     3→create table if not exists video_tasks (
     4→  video_id text primary key,
     5→  language text not null,
     6→  r2_tar_key text not null,
     7→  status text not null default 'pending', -- pending|claimed|processing|uploading|completed|failed
     8→  shard_id uuid,
     9→  claimed_by text,
    10→  lease_expires_at timestamptz,
    11→  attempts int not null default 0,
    12→  last_error text,
    13→  created_at timestamptz default now(),
    14→  updated_at timestamptz default now()
    15→);
    16→
    17→create index if not exists video_tasks_language_status_idx
    18→  on video_tasks(language, status);
    19→
    20→create table if not exists shard_runs (
    21→  shard_id uuid primary key,
    22→  language text not null,
    23→  target_video_count int not null,
    24→  assigned_video_count int not null,
    25→  status text not null default 'claimed', -- claimed|processing|uploading|completed|failed|stale
    26→  claimed_by text,
    27→  lease_expires_at timestamptz,
    28→  shard_prefix text,
    29→  output_format text not null default 'flac',
    30→  metrics jsonb,
    31→  created_at timestamptz default now(),
    32→  updated_at timestamptz default now()
    33→);
    34→
    35→create index if not exists shard_runs_language_status_idx
    36→  on shard_runs(language, status);
    37→
    38→create table if not exists worker_heartbeats_v2 (
    39→  worker_id text primary key,
    40→  hostname text,
    41→  instance_type text,
    42→  language text,
    43→  status text not null default 'idle', -- idle|claiming|processing|uploading|failed
    44→  shard_id uuid,
    45→  videos_assigned int default 0,
    46→  videos_processed int default 0,
    47→  segments_processed int default 0,
    48→  outputs_generated int default 0,
    49→  last_error text,
    50→  last_heartbeat timestamptz default now()
    51→);
    52→
    53→create table if not exists video_task_events (
    54→  id bigserial primary key,
    55→  video_id text,
    56→  shard_id uuid,
    57→  worker_id text,
    58→  event text,
    59→  details jsonb,
    60→  created_at timestamptz default now()
    61→);
    62→
    63→create table if not exists language_queue (
    64→  language text primary key,
    65→  priority int not null default 0,
    66→  status text not null default 'pending', -- pending|processing|completed|stale
    67→  claimed_by text,
    68→  lease_expires_at timestamptz,
    69→  created_at timestamptz default now(),
    70→  updated_at timestamptz default now(),
    71→  completed_at timestamptz
    72→);
    73→
    74→create table if not exists language_state (
    75→  id int primary key,
    76→  active_language text,
    77→  updated_at timestamptz default now()
    78→);
    79→
    80→insert into language_state (id, active_language)
    81→values (1, null)
    82→on conflict (id) do nothing;
    83→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

-- Supabase schema for horizontal workers

create table if not exists video_tasks (
  video_id text primary key,
  language text not null,
  r2_tar_key text not null,
  status text not null default 'pending', -- pending|claimed|processing|uploading|completed|failed
  shard_id uuid,
  claimed_by text,
  lease_expires_at timestamptz,
  attempts int not null default 0,
  last_error text,
  created_at timestamptz default now(),
  updated_at timestamptz default now()
);

create index if not exists video_tasks_language_status_idx
  on video_tasks(language, status);

create table if not exists shard_runs (
  shard_id uuid primary key,
  language text not null,
  target_video_count int not null,
  assigned_video_count int not null,
  status text not null default 'claimed', -- claimed|processing|uploading|completed|failed|stale
  claimed_by text,
  lease_expires_at timestamptz,
  shard_prefix text,
  output_format text not null default 'flac',
  metrics jsonb,
  created_at timestamptz default now(),
  updated_at timestamptz default now()
);

create index if not exists shard_runs_language_status_idx
  on shard_runs(language, status);

create table if not exists worker_heartbeats_v2 (
  worker_id text primary key,
  hostname text,
  instance_type text,
  language text,
  status text not null default 'idle', -- idle|claiming|processing|uploading|failed
  shard_id uuid,
  videos_assigned int default 0,
  videos_processed int default 0,
  segments_processed int default 0,
  outputs_generated int default 0,
  last_error text,
  last_heartbeat timestamptz default now()
);

create table if not exists video_task_events (
  id bigserial primary key,
  video_id text,
  shard_id uuid,
  worker_id text,
  event text,
  details jsonb,
  created_at timestamptz default now()
);

create table if not exists language_queue (
  language text primary key,
  priority int not null default 0,
  status text not null default 'pending', -- pending|processing|completed|stale
  claimed_by text,
  lease_expires_at timestamptz,
  created_at timestamptz default now(),
  updated_at timestamptz default now(),
  completed_at timestamptz
);

create table if not exists language_state (
  id int primary key,
  active_language text,
  updated_at timestamptz default now()
);

insert into language_state (id, active_language)
values (1, null)
on conflict (id) do nothing;