Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""R2 upload module with retry logic and progress tracking."""

from __future__ import annotations

import logging
import os
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, Optional

import boto3
from boto3.s3.transfer import TransferConfig
from botocore.exceptions import ClientError

logger = logging.getLogger(__name__)

@dataclass
class UploadResult:
    """Result of an upload operation."""

    success: bool = True
    uploaded_keys: list[str] = field(default_factory=list)
    failed_files: list[str] = field(default_factory=list)
    total_bytes: int = 0
    elapsed_sec: float = 0.0
    errors: list[str] = field(default_factory=list)

class R2Uploader:
    """Handles uploads to R2 with retry logic and progress tracking."""

    def __init__(
        self,
        endpoint_url: str,
        access_key_id: str,
        secret_access_key: str,
        bucket: str,
        max_retries: int = 3,
        retry_delay_sec: float = 5.0,
        max_concurrency: int = 4,
    ):
        self.bucket = bucket
        self.max_retries = max_retries
        self.retry_delay_sec = retry_delay_sec

        self.client = boto3.client(
            "s3",
            endpoint_url=endpoint_url,
            aws_access_key_id=access_key_id,
            aws_secret_access_key=secret_access_key,
        )

        self.transfer_config = TransferConfig(
            multipart_threshold=8 * 1024 * 1024,  # 8MB
            multipart_chunksize=64 * 1024 * 1024,  # 64MB
            max_concurrency=max_concurrency,
            use_threads=True,
        )

        # Progress callback
        self._on_progress: Optional[Callable[[str, int, int], None]] = None

    def set_progress_callback(
        self, callback: Optional[Callable[[str, int, int], None]]
    ) -> None:
        """Set callback for upload progress: callback(filename, bytes_transferred, total_bytes)."""
        self._on_progress = callback

    def upload_file(
        self,
        local_path: Path,
        remote_key: str,
        metadata: Optional[dict[str, str]] = None,
    ) -> bool:
        """Upload a single file with retry logic.

        Args:
            local_path: Path to local file
            remote_key: Remote key in bucket
            metadata: Optional metadata to attach

        Returns:
            True if upload succeeded, False otherwise
        """
        if not local_path.exists():
            logger.error(f"File not found: {local_path}")
            return False

        file_size = local_path.stat().st_size

        for attempt in range(self.max_retries):
            try:
                extra_args = {}
                if metadata:
                    extra_args["Metadata"] = metadata

                # Create progress callback for this file
                callback = None
                if self._on_progress:
                    def make_callback(filename: str, total: int):
                        bytes_transferred = 0

                        def progress(chunk_bytes: int):
                            nonlocal bytes_transferred
                            bytes_transferred += chunk_bytes
                            if self._on_progress:
                                self._on_progress(filename, bytes_transferred, total)

                        return progress

                    callback = make_callback(local_path.name, file_size)

                self.client.upload_file(
                    str(local_path),
                    self.bucket,
                    remote_key,
                    Config=self.transfer_config,
                    ExtraArgs=extra_args if extra_args else None,
                    Callback=callback,
                )

                logger.debug(f"Uploaded {local_path.name} -> {remote_key}")
                return True

            except ClientError as e:
                error_code = e.response.get("Error", {}).get("Code", "Unknown")
                logger.warning(
                    f"Upload attempt {attempt + 1}/{self.max_retries} failed for "
                    f"{local_path.name}: {error_code} - {e}"
                )

                if attempt < self.max_retries - 1:
                    delay = self.retry_delay_sec * (2 ** attempt)  # Exponential backoff
                    logger.info(f"Retrying in {delay:.1f}s...")
                    time.sleep(delay)

            except Exception as e:
                logger.error(f"Unexpected error uploading {local_path.name}: {e}")
                if attempt < self.max_retries - 1:
                    time.sleep(self.retry_delay_sec)

        logger.error(f"Failed to upload {local_path} after {self.max_retries} attempts")
        return False

    def upload_directory(
        self,
        local_dir: Path,
        remote_prefix: str,
        pattern: str = "*",
    ) -> UploadResult:
        """Upload all files from a directory.

        Args:
            local_dir: Local directory to upload from
            remote_prefix: Remote prefix for all files
            pattern: Glob pattern to match files

        Returns:
            UploadResult with details
        """
        result = UploadResult()
        start_time = time.time()

        if not local_dir.exists():
            result.success = False
            result.errors.append(f"Directory not found: {local_dir}")
            return result

        files = list(local_dir.glob(pattern))
        if not files:
            logger.warning(f"No files matching '{pattern}' in {local_dir}")
            return result

        for local_path in sorted(files):
            if not local_path.is_file():
                continue

            remote_key = f"{remote_prefix.rstrip('/')}/{local_path.name}"
            file_size = local_path.stat().st_size

            if self.upload_file(local_path, remote_key):
                result.uploaded_keys.append(remote_key)
                result.total_bytes += file_size
            else:
                result.failed_files.append(str(local_path))

        result.elapsed_sec = time.time() - start_time
        result.success = len(result.failed_files) == 0

        logger.info(
            f"Directory upload complete: {len(result.uploaded_keys)} files, "
            f"{result.total_bytes / (1024**2):.1f}MB in {result.elapsed_sec:.1f}s"
        )

        return result

    def upload_shard(
        self,
        output_dir: Path,
        language: str,
        shard_id: str,
        audio_ext: str = "flac",
    ) -> UploadResult:
        """Upload a complete shard (tar + jsonl for each sample rate).

        This uploads:
        - {language}_16/shard-{shard_id}.tar and .jsonl
        - {language}_24/shard-{shard_id}.tar and .jsonl (if exists)
        - {language}_32/shard-{shard_id}.tar and .jsonl (if exists)
        - {language}_44/shard-{shard_id}.tar and .jsonl (if exists)

        Args:
            output_dir: Base output directory
            language: Language code
            shard_id: Shard identifier
            audio_ext: Audio extension (flac or wav)

        Returns:
            UploadResult with details
        """
        result = UploadResult()
        start_time = time.time()

        sr_map = {
            16000: "16",
            24000: "24",
            32000: "32",
            44100: "44",
        }

        for sr, sr_suffix in sr_map.items():
            local_dir = output_dir / f"{language}_{sr_suffix}"
            if not local_dir.exists():
                continue

            # Upload tar file
            tar_pattern = f"*-{shard_id}*.tar"
            for tar_file in local_dir.glob(tar_pattern):


Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""Configuration management for workers."""

from __future__ import annotations

import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional

@dataclass
class WorkerConfig:
    """Worker configuration with defaults and environment overrides."""

    # Worker identity
    worker_id: str = ""
    language: str = ""
    auto_language: bool = False

    # Processing settings
    target_videos: int = 100  # Videos per shard (approx 5GB)
    lease_minutes: int = 60  # Lease duration for claims
    heartbeat_interval_sec: int = 15  # Heartbeat frequency
    max_attempts: int = 5  # Retry attempts before skipping

    # Directory settings
    work_dir: Path = field(default_factory=lambda: Path("/tmp/worker"))
    output_dir: Path = field(default_factory=lambda: Path("/tmp/worker_output"))

    # Storage management
    max_disk_usage_gb: float = 150.0  # Max disk usage before cleanup
    cleanup_threshold_gb: float = 100.0  # Target after cleanup
    cleanup_check_interval_sec: int = 60  # Storage check frequency

    # Parallelism
    download_workers: int = 0  # 0 = auto
    segment_workers: int = 0  # 0 = auto
    upload_workers: int = 0  # 0 = auto

    # Output settings
    output_format: str = "flac"  # flac or wav
    target_shard_size_mb: int = 0  # 0 disables size-based splitting

    # Processing options
    chunk_enabled: bool = True
    chunk_min_sec: float = 6.0
    chunk_max_sec: float = 10.0
    chunk_threshold_sec: float = 12.0
    chunk_all_buckets: bool = False

    silence_enabled: bool = True
    silence_frame_ms: float = 50.0
    silence_hop_ms: float = 10.0
    silence_search_sec: float = 0.5
    silence_percentile: float = 20.0
    silence_db_drop: float = 20.0

    val_ratio: float = 0.01
    seed: int = 17

    # Retry settings
    max_retries: int = 3
    retry_delay_sec: float = 5.0

    # Run mode
    once: bool = False  # Process single shard and exit
    dry_run: bool = False  # Don't actually process

    @classmethod
    def from_env(cls) -> "WorkerConfig":
        """Create config from environment variables."""
        return cls(
            worker_id=os.environ.get("WORKER_ID", ""),
            language=os.environ.get("LANGUAGE", ""),
            auto_language=os.environ.get("AUTO_LANGUAGE", "").lower()
            in ("1", "true", "yes"),
            target_videos=int(os.environ.get("TARGET_VIDEOS", "100")),
            lease_minutes=int(os.environ.get("LEASE_MINUTES", "60")),
            heartbeat_interval_sec=int(os.environ.get("HEARTBEAT_INTERVAL_SEC", "15")),
            max_attempts=int(os.environ.get("MAX_ATTEMPTS", "5")),
            work_dir=Path(os.environ.get("WORK_DIR", "/tmp/worker")),
            output_dir=Path(os.environ.get("OUTPUT_DIR", "/tmp/worker_output")),
            max_disk_usage_gb=float(os.environ.get("MAX_DISK_USAGE_GB", "150")),
            cleanup_threshold_gb=float(os.environ.get("CLEANUP_THRESHOLD_GB", "100")),
            cleanup_check_interval_sec=int(os.environ.get("CLEANUP_CHECK_INTERVAL_SEC", "60")),
            download_workers=int(os.environ.get("DOWNLOAD_WORKERS", "0")),
            segment_workers=int(os.environ.get("SEGMENT_WORKERS", "0")),
            upload_workers=int(os.environ.get("UPLOAD_WORKERS", "0")),
            output_format=os.environ.get("OUTPUT_FORMAT", "flac"),
            target_shard_size_mb=int(
                os.environ.get("TARGET_SHARD_SIZE_MB", os.environ.get("SHARD_SIZE_MB", "0"))
            ),
            max_retries=int(os.environ.get("MAX_RETRIES", "3")),
            retry_delay_sec=float(os.environ.get("RETRY_DELAY_SEC", "5")),
            once=os.environ.get("ONCE", "").lower() in ("1", "true", "yes"),
            dry_run=os.environ.get("DRY_RUN", "").lower() in ("1", "true", "yes"),
        )

@dataclass
class EnvConfig:
    """Environment configuration from .env file."""

    # Database
    database_url: str = ""

    # R2 Storage
    r2_endpoint_url: str = ""
    r2_access_key_id: str = ""
    r2_secret_access_key: str = ""
    r2_cleaned_bucket: str = ""  # Source bucket with video tars
    r2_bucket_shards: str = ""  # Destination bucket for shards
    r2_prefix: Optional[str] = None

    @classmethod
    def from_file(cls, env_path: Path) -> "EnvConfig":
        """Load from .env file."""
        env: dict[str, str] = {}
        if env_path.exists():
            for line in env_path.read_text().splitlines():
                stripped = line.strip()
                if not stripped or stripped.startswith("#") or "=" not in stripped:
                    continue
                if stripped.startswith("export "):
                    stripped = stripped[len("export "):]
                key, value = stripped.split("=", 1)
                env[key.strip()] = value.strip().strip('"').strip("'")

        return cls(
            database_url=env.get("DATABASE_URL", ""),
            r2_endpoint_url=env.get("R2_ENDPOINT_URL", ""),
            r2_access_key_id=env.get("R2_ACCESS_KEY_ID", ""),
            r2_secret_access_key=env.get("R2_SECRET_ACCESS_KEY", ""),
            r2_cleaned_bucket=env.get("R2_CLEANED_BUCKET") or env.get("R2_ClEANED_BUCKET", ""),
            r2_bucket_shards=env.get("R2_BUCKET_SHARDS")
            or env.get("R2_PROCESSED_BUCKET", ""),
            r2_prefix=env.get("R2_PREFIX"),
        )

    @classmethod
    def from_env_vars(cls) -> "EnvConfig":
        """Load from environment variables."""
        return cls(
            database_url=os.environ.get("DATABASE_URL", ""),
            r2_endpoint_url=os.environ.get("R2_ENDPOINT_URL", ""),
            r2_access_key_id=os.environ.get("R2_ACCESS_KEY_ID", ""),
            r2_secret_access_key=os.environ.get("R2_SECRET_ACCESS_KEY", ""),
            r2_cleaned_bucket=os.environ.get("R2_CLEANED_BUCKET") or os.environ.get("R2_ClEANED_BUCKET", ""),
            r2_bucket_shards=os.environ.get("R2_BUCKET_SHARDS")
            or os.environ.get("R2_PROCESSED_BUCKET", ""),
            r2_prefix=os.environ.get("R2_PREFIX"),
        )

    def validate(self) -> list[str]:
        """Validate configuration, return list of errors."""
        errors = []
        if not self.database_url:
            errors.append("DATABASE_URL is required")
        if not self.r2_endpoint_url:
            errors.append("R2_ENDPOINT_URL is required")
        if not self.r2_access_key_id:
            errors.append("R2_ACCESS_KEY_ID is required")
        if not self.r2_secret_access_key:
            errors.append("R2_SECRET_ACCESS_KEY is required")
        if not self.r2_cleaned_bucket:
            errors.append("R2_CLEANED_BUCKET is required")
        if not self.r2_bucket_shards:
            errors.append("R2_BUCKET_SHARDS is required")
        return errors


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Continuing detailed scan of processing pipeline**

I'll focus on scanning key sections of worker_processing to capture audio parsing, metric computation, chunking, resampling, filtering, and output handling, and check for PANN usage for music detection next.


rg -n "pann|music|pyannote|vad|speaker" workers/worker_processing.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
227:def compute_quality_score(metrics: dict[str, float], has_music: bool) -> float:
232:        + 0.2 * (1.0 if not has_music else 0.5)
399:def make_segment_seed(base_seed: int, video_id: str, speaker: str, index: int) -> int:
401:    key = f"{video_id}-{speaker}-{index}-{base_seed}"
406:    """Parse segment filename to extract speaker and start time."""
416:    speaker = "_".join(parts[:-2])
427:    return speaker, start_time
435:    speaker: str,
463:    music_stats = segment_meta.get("music_stats") or {}
464:    has_music = bool(music_stats.get("has_music", False))
491:        sampling_weight = base_weight * (0.5 if has_music else 1.0)
492:        quality_score = compute_quality_score(metrics, has_music)
517:            "music_prob": float(music_stats.get("music_mean", 0.0)),
518:            "music_stats": music_stats,
519:            "speaker": speaker,
520:            "vad_speech_regions": [[0.0, slice_duration]],
537:    sample_id_base = f"{video_id}__{speaker}_{index:04d}"
867:                        (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
878:                        speaker, start_time = parsed
879:                        seg_meta = segment_map.get((speaker, round(start_time, 2)))
883:                                if s.get("speaker") == speaker
899:                        rng_seed = make_segment_seed(self.seed, video_id, speaker, index)
907:                            speaker,


Summary: auto


**Expanding context around line 350**


sed -n '240,560p' workers/worker_processing.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
    """Resample audio to target sample rate."""
    if src_sr == dst_sr:
        return audio
    gcd = math.gcd(src_sr, dst_sr)
    up = dst_sr // gcd
    down = src_sr // gcd
    return signal.resample_poly(audio, up, down).astype(np.float32, copy=False)

def encode_audio(audio: np.ndarray, sr: int, output_format: str) -> bytes:
    """Encode audio to bytes."""
    buf = io.BytesIO()
    if output_format == "wav":
        sf.write(buf, audio, sr, format="WAV", subtype="PCM_16")
    else:
        sf.write(buf, audio, sr, format="FLAC", subtype="PCM_16")
    return buf.getvalue()

def duration_bucket(duration_sec: float) -> Optional[tuple[str, float]]:
    """Get duration bucket and weight."""
    if duration_sec < 3.0:
        return None
    if duration_sec < 6.0:
        return "3-6s", 0.3
    if duration_sec < 12.0:
        return "6-12s", 1.0
    if duration_sec < 24.0:
        return "12-24s", 0.8
    if duration_sec <= 60.0:
        return "24-60s", 0.5
    return None

def compute_frame_rms(
    audio: np.ndarray, sr: int, frame_sec: float, hop_sec: float
) -> tuple[np.ndarray, np.ndarray]:
    """Compute frame-level RMS for silence detection."""
    frame_len = max(1, int(round(frame_sec * sr)))
    hop_len = max(1, int(round(hop_sec * sr)))
    if audio.size < frame_len:
        return np.array([], dtype=np.float32), np.array([], dtype=np.float32)

    if not audio.flags["C_CONTIGUOUS"]:
        audio = np.ascontiguousarray(audio)

    num_frames = 1 + (audio.size - frame_len) // hop_len
    if num_frames <= 0:
        return np.array([], dtype=np.float32), np.array([], dtype=np.float32)

    stride = audio.strides[0]
    frames = np.lib.stride_tricks.as_strided(
        audio,
        shape=(num_frames, frame_len),
        strides=(hop_len * stride, stride),
    )
    rms = np.sqrt(np.mean(frames * frames, axis=1) + 1e-12)
    times = (np.arange(num_frames, dtype=np.float32) * hop_len + (frame_len / 2)) / sr
    return times.astype(np.float32), rms.astype(np.float32)

def find_silence_boundary(
    times: np.ndarray,
    rms: np.ndarray,
    search_start: float,
    search_end: float,
    threshold_rms: float,
    median_db: float,
    db_drop: float,
) -> Optional[float]:
    """Find optimal silence boundary for chunking."""
    if times.size == 0:
        return None
    mask = (times >= search_start) & (times <= search_end)
    if not np.any(mask):
        return None
    subset = rms[mask]
    min_idx = int(np.argmin(subset))
    min_rms = float(subset[min_idx])
    min_time = float(times[mask][min_idx])
    min_db = 20.0 * math.log10(min_rms + 1e-12)
    if min_rms <= threshold_rms or (median_db - min_db) >= db_drop:
        return min_time
    return None

def make_chunk_windows(
    audio: np.ndarray,
    sr: int,
    cfg: ChunkConfig,
    silence_cfg: SilenceConfig,
    rng: random.Random,
) -> list[tuple[float, float]]:
    """Generate chunk windows for a segment."""
    duration_sec = float(audio.shape[0] / sr)
    if not cfg.enabled or duration_sec < cfg.threshold_sec:
        return [(0.0, duration_sec)]

    if not silence_cfg.enabled:
        windows: list[tuple[float, float]] = []
        offset = 0.0
        while duration_sec - offset >= cfg.min_sec:
            remaining = duration_sec - offset
            if remaining <= cfg.max_sec:
                length = remaining
            else:
                length = rng.uniform(cfg.min_sec, cfg.max_sec)
                if remaining - length < cfg.min_sec:
                    length = remaining
            windows.append((offset, length))
            offset += length
        return windows

    times, rms = compute_frame_rms(
        audio, sr, silence_cfg.frame_sec, silence_cfg.hop_sec
    )
    if times.size == 0:
        return [(0.0, duration_sec)]

    threshold_rms = float(np.quantile(rms, silence_cfg.percentile / 100.0))
    median_db = 20.0 * math.log10(float(np.median(rms)) + 1e-12)

    windows: list[tuple[float, float]] = []
    offset = 0.0
    while duration_sec - offset > cfg.max_sec:
        remaining = duration_sec - offset
        latest_cut = duration_sec - cfg.min_sec
        if offset + cfg.min_sec >= latest_cut:
            break
        target_len = rng.uniform(cfg.min_sec, cfg.max_sec)
        if remaining - target_len < cfg.min_sec:
            target_len = remaining - cfg.min_sec
        target_end = offset + target_len
        search_low = max(offset + cfg.min_sec, target_end - silence_cfg.search_sec)
        search_high = min(offset + cfg.max_sec, target_end + silence_cfg.search_sec, latest_cut)
        if search_high <= search_low:
            search_low = offset + cfg.min_sec
            search_high = min(offset + cfg.max_sec, latest_cut)
        boundary = find_silence_boundary(
            times,
            rms,
            search_low,
            search_high,
            threshold_rms,
            median_db,
            silence_cfg.db_drop,
        )
        if boundary is None:
            if not windows:
                return [(0.0, duration_sec)]
            windows.append((offset, duration_sec - offset))
            return windows
        windows.append((offset, boundary - offset))
        offset = boundary

    windows.append((offset, duration_sec - offset))
    return windows

def make_segment_seed(base_seed: int, video_id: str, speaker: str, index: int) -> int:
    """Generate deterministic seed for a segment."""
    key = f"{video_id}-{speaker}-{index}-{base_seed}"
    return zlib.adler32(key.encode("utf-8")) & 0xFFFFFFFF

def parse_segment_filename(filename: str) -> Optional[tuple[str, float]]:
    """Parse segment filename to extract speaker and start time."""
    base = filename.split("/")[-1]
    if not base.endswith(".flac"):
        return None

    base = base[:-5]
    parts = base.split("_")
    if len(parts) < 3:
        return None

    speaker = "_".join(parts[:-2])
    time_range = parts[-1]
    if "-" not in time_range:
        return None

    try:
        start_str = time_range.split("-")[0]
        start_time = float(start_str)
    except (ValueError, IndexError):
        return None

    return speaker, start_time

def process_segment(
    video_id: str,
    language: str,
    segment_bytes: bytes,
    segment_meta: dict[str, Any],
    speaker: str,
    index: int,
    chunk_cfg: ChunkConfig,
    silence_cfg: SilenceConfig,
    rng_seed: int,
    output_format: str,
) -> tuple[list[SegmentOutput], dict[str, int]]:
    """Process a single segment and return outputs."""
    stats = {
        "dropped_too_short": 0,
        "dropped_too_long": 0,
        "dropped_clipped": 0,
        "chunks_emitted": 0,
        "segments_chunked": 0,
    }
    outputs: list[SegmentOutput] = []

    try:
        audio, sr = sf.read(io.BytesIO(segment_bytes), dtype="float32")
    except Exception:
        return outputs, stats

    if audio.ndim > 1:
        audio = np.mean(audio, axis=1)

    rng = random.Random(rng_seed)
    windows = make_chunk_windows(audio, sr, chunk_cfg, silence_cfg, rng)

    music_stats = segment_meta.get("music_stats") or {}
    has_music = bool(music_stats.get("has_music", False))

    def emit_outputs(
        audio_slice: np.ndarray,
        metrics: dict[str, float],
        sample_id: str,
        segment_index: int,
        chunk_index: Optional[int],
        start: Optional[float],
        end: Optional[float],
        original_start: Optional[float],
        original_end: Optional[float],
        target_srs: list[int],
    ) -> None:
        slice_duration = float(audio_slice.shape[0] / sr)
        bucket = duration_bucket(slice_duration)
        if bucket is None:
            if slice_duration < 3.0:
                stats["dropped_too_short"] += 1
            else:
                stats["dropped_too_long"] += 1
            return
        if metrics["clipping_fraction"] > 0.1:
            stats["dropped_clipped"] += 1
            return

        duration_bucket_name, base_weight = bucket
        sampling_weight = base_weight * (0.5 if has_music else 1.0)
        quality_score = compute_quality_score(metrics, has_music)
        effective_nyquist = metrics["effective_nyquist_hz"]

        base_metadata = {
            "id": sample_id,
            "video_id": video_id,
            "language": language,
            "segment_index": segment_index,
            "chunk_index": chunk_index,
            "start": start,
            "end": end,
            "original_start": original_start,
            "original_end": original_end,
            "duration_sec": slice_duration,
            "native_sr": sr,
            "effective_nyquist_hz": effective_nyquist,
            "spectral_rolloff_hz": metrics["spectral_rolloff_hz"],
            "high_freq_energy_ratio": metrics["high_freq_energy_ratio"],
            "is_truly_wideband": bool(effective_nyquist >= 12000),
            "native_sr_bucket": classify_native_bucket(effective_nyquist),
            "loudness_lufs": metrics["loudness_lufs"],
            "loudness_rms_db": metrics["loudness_rms_db"],
            "clipping_fraction": metrics["clipping_fraction"],
            "snr_proxy": metrics["snr_proxy"],
            "quality_score": quality_score,
            "music_prob": float(music_stats.get("music_mean", 0.0)),
            "music_stats": music_stats,
            "speaker": speaker,
            "vad_speech_regions": [[0.0, slice_duration]],
            "duration_bucket": duration_bucket_name,
            "sampling_weight": sampling_weight,
        }

        for target_sr in target_srs:
            resampled = resample_audio(audio_slice, sr, target_sr)
            resampled = np.clip(resampled, -1.0, 1.0)
            audio_bytes = encode_audio(resampled, target_sr, output_format)
            meta = dict(base_metadata)
            meta["sample_rate"] = target_sr
            meta["resample_target_sr"] = target_sr
            outputs.append(SegmentOutput(target_sr, audio_bytes, meta))

        if target_srs == [16000]:
            stats["chunks_emitted"] += 1

    sample_id_base = f"{video_id}__{speaker}_{index:04d}"
    use_chunk_suffix = len(windows) > 1
    if use_chunk_suffix:
        stats["segments_chunked"] += 1

    segment_start = segment_meta.get("start")
    segment_end = segment_meta.get("end")
    original_start = segment_meta.get("original_start")
    original_end = segment_meta.get("original_end")

    # Emit 16k outputs for all chunks
    for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
        start_idx = int(round(offset_sec * sr))
        end_idx = int(round((offset_sec + chunk_len_sec) * sr))
        audio_slice = audio[start_idx:end_idx]
        if audio_slice.size == 0:
            continue
        slice_duration = float(audio_slice.shape[0] / sr)
        slice_metrics = compute_audio_metrics(audio_slice, sr)

        chunk_start = float(segment_start) + offset_sec if segment_start is not None else None
        chunk_end = chunk_start + slice_duration if chunk_start is not None else None
        chunk_original_start = (
            float(original_start) + offset_sec if original_start is not None else None


sed -n '560,980p' workers/worker_processing.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
            float(original_start) + offset_sec if original_start is not None else None
        )
        chunk_original_end = (
            chunk_original_start + slice_duration
            if chunk_original_start is not None
            else None
        )

        sample_id = sample_id_base
        chunk_index = None
        if use_chunk_suffix:
            sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
            chunk_index = chunk_idx

        emit_outputs(
            audio_slice,
            slice_metrics,
            sample_id,
            index,
            chunk_index,
            chunk_start,
            chunk_end,
            chunk_original_start,
            chunk_original_end,
            [16000],
        )

    # High-rate buckets for full segment
    full_metrics = compute_audio_metrics(audio, sr)
    effective_nyquist = full_metrics["effective_nyquist_hz"]
    targets_high: list[int] = []
    if effective_nyquist >= 8000:
        targets_high.append(24000)
    if effective_nyquist >= 12000:
        targets_high.append(32000)
    if effective_nyquist >= 16000:
        targets_high.append(44100)

    if targets_high:
        if chunk_cfg.all_buckets and use_chunk_suffix:
            for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
                start_idx = int(round(offset_sec * sr))
                end_idx = int(round((offset_sec + chunk_len_sec) * sr))
                audio_slice = audio[start_idx:end_idx]
                if audio_slice.size == 0:
                    continue
                slice_duration = float(audio_slice.shape[0] / sr)
                slice_metrics = compute_audio_metrics(audio_slice, sr)
                slice_effective_nyquist = slice_metrics["effective_nyquist_hz"]
                slice_targets: list[int] = []
                if slice_effective_nyquist >= 8000:
                    slice_targets.append(24000)
                if slice_effective_nyquist >= 12000:
                    slice_targets.append(32000)
                if slice_effective_nyquist >= 16000:
                    slice_targets.append(44100)
                if not slice_targets:
                    continue

                chunk_start = (
                    float(segment_start) + offset_sec if segment_start is not None else None
                )
                chunk_end = (
                    chunk_start + slice_duration if chunk_start is not None else None
                )
                chunk_original_start = (
                    float(original_start) + offset_sec if original_start is not None else None
                )
                chunk_original_end = (
                    chunk_original_start + slice_duration
                    if chunk_original_start is not None
                    else None
                )

                sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
                emit_outputs(
                    audio_slice,
                    slice_metrics,
                    sample_id,
                    index,
                    chunk_idx,
                    chunk_start,
                    chunk_end,
                    chunk_original_start,
                    chunk_original_end,
                    slice_targets,
                )
        else:
            emit_outputs(
                audio,
                full_metrics,
                sample_id_base,
                index,
                None,
                segment_start,
                segment_end,
                original_start,
                original_end,
                targets_high,
            )

    return outputs, stats

class ShardWriter:
    """Write samples to tar + jsonl shards."""

    def __init__(
        self, output_dir: Path, prefix: str, target_size_bytes: int, audio_ext: str
    ) -> None:
        self.output_dir = output_dir
        self.prefix = prefix
        self.target_size_bytes = target_size_bytes
        self.audio_ext = audio_ext
        self.shard_index = 0
        self.current_tar: Optional[tarfile.TarFile] = None
        self.current_jsonl = None
        self.current_size = 0
        self.current_count = 0
        self.current_name = ""
        self.output_dir.mkdir(parents=True, exist_ok=True)

    def _open_new_shard(self) -> None:
        if self.current_tar:
            self.close()
        shard_name = f"{self.prefix}-{self.shard_index:06d}"
        tar_path = self.output_dir / f"{shard_name}.tar"
        jsonl_path = self.output_dir / f"{shard_name}.jsonl"
        self.current_tar = tarfile.open(tar_path, "w")
        self.current_jsonl = jsonl_path.open("w", encoding="utf-8")
        self.current_size = 0
        self.current_count = 0
        self.current_name = shard_name
        self.shard_index += 1

    def add_sample(self, sample_id: str, audio_bytes: bytes, metadata: dict[str, Any]) -> str:
        if self.current_tar is None:
            self._open_new_shard()
        if (
            self.target_size_bytes > 0
            and self.current_count > 0
            and self.current_size + len(audio_bytes) > self.target_size_bytes
        ):
            self._open_new_shard()

        assert self.current_tar is not None
        assert self.current_jsonl is not None

        arcname = f"{sample_id}.{self.audio_ext}"
        info = tarfile.TarInfo(name=arcname)
        info.size = len(audio_bytes)
        info.mtime = int(time.time())
        self.current_tar.addfile(info, io.BytesIO(audio_bytes))
        self.current_jsonl.write(json.dumps(metadata) + "\n")

        self.current_size += len(audio_bytes)
        self.current_count += 1
        return self.current_name

    def close(self) -> None:
        if self.current_tar:
            self.current_tar.close()
        if self.current_jsonl:
            self.current_jsonl.close()
        self.current_tar = None
        self.current_jsonl = None

class VideoProcessor:
    """Process videos and write to shards."""

    def __init__(
        self,
        output_dir: Path,
        language: str,
        shard_prefix: str,
        chunk_cfg: ChunkConfig,
        silence_cfg: SilenceConfig,
        output_format: str = "flac",
        target_shard_size_mb: int = 5120,
        segment_workers: int = 8,
        seed: int = 17,
        val_ratio: float = 0.01,
        on_progress: Optional[Callable[[int, int], None]] = None,
    ):
        self.output_dir = output_dir
        self.language = language
        self.shard_prefix = shard_prefix
        self.chunk_cfg = chunk_cfg
        self.silence_cfg = silence_cfg
        self.output_format = output_format
        self.seed = seed
        self.val_ratio = val_ratio
        self.segment_workers = segment_workers
        self.on_progress = on_progress

        audio_ext = "wav" if output_format == "wav" else "flac"
        target_bytes = max(0, target_shard_size_mb) * 1024 * 1024

        self.writers = {
            16000: ShardWriter(
                output_dir / f"{language}_16", shard_prefix, target_bytes, audio_ext
            ),
            24000: ShardWriter(
                output_dir / f"{language}_24", shard_prefix, target_bytes, audio_ext
            ),
            32000: ShardWriter(
                output_dir / f"{language}_32", shard_prefix, target_bytes, audio_ext
            ),
            44100: ShardWriter(
                output_dir / f"{language}_44", shard_prefix, target_bytes, audio_ext
            ),
        }

        # Manifests
        manifest_dir = output_dir / "manifests"
        manifest_dir.mkdir(parents=True, exist_ok=True)
        self.train_manifest = (manifest_dir / f"{language}_train_{shard_prefix}.jsonl").open(
            "w", encoding="utf-8"
        )
        self.val_manifest = (manifest_dir / f"{language}_val_{shard_prefix}.jsonl").open(
            "w", encoding="utf-8"
        )

        self.stats = ProcessingStats()
        self.audio_ext = audio_ext

    def process_videos(
        self,
        download_paths: dict[str, Path],
        val_ids: Optional[set[str]] = None,
    ) -> ProcessingStats:
        """Process all videos and write to shards."""
        if val_ids is None:
            # Determine val set
            video_ids = list(download_paths.keys())
            random.seed(self.seed)
            val_count = max(1, int(len(video_ids) * self.val_ratio)) if len(video_ids) > 1 else 0
            val_ids = set(random.sample(video_ids, val_count)) if val_count > 0 else set()

        max_inflight = max(128, self.segment_workers * 2)
        pending: set[Any] = set()
        future_info: dict[Any, bool] = {}

        def handle_future(future) -> None:
            is_val = future_info.pop(future, False)
            outputs, seg_stats = future.result()
            self.stats.dropped_too_short += seg_stats["dropped_too_short"]
            self.stats.dropped_too_long += seg_stats["dropped_too_long"]
            self.stats.dropped_clipped += seg_stats["dropped_clipped"]
            self.stats.segments_chunked += seg_stats.get("segments_chunked", 0)
            self.stats.chunks_emitted += seg_stats.get("chunks_emitted", 0)

            for out in outputs:
                self.stats.segments_kept[out.sample_rate] += 1
                if out.sample_rate == 16000:
                    bucket = out.metadata.get("native_sr_bucket")
                    if bucket:
                        self.stats.native_sr_buckets[bucket] = (
                            self.stats.native_sr_buckets.get(bucket, 0) + 1
                        )
                writer = self.writers[out.sample_rate]
                shard_name = writer.add_sample(
                    out.metadata["id"], out.audio_bytes, out.metadata
                )
                manifest_entry = dict(out.metadata)
                manifest_entry["shard"] = f"{shard_name}.tar"
                manifest_entry["audio"] = f"{out.metadata['id']}.{self.audio_ext}"
                line = json.dumps(manifest_entry) + "\n"
                if is_val:
                    self.val_manifest.write(line)
                else:
                    self.train_manifest.write(line)

        def drain_completed() -> None:
            nonlocal pending
            if not pending:
                return
            done, pending = wait(pending, return_when=FIRST_COMPLETED)
            for future in done:
                handle_future(future)

        total_videos = len(download_paths)
        processed_videos = 0

        with ThreadPoolExecutor(max_workers=self.segment_workers) as executor:
            for video_id, tar_path in download_paths.items():
                self.stats.videos += 1
                is_val = video_id in val_ids

                try:
                    tf = tarfile.open(tar_path, "r")
                except Exception as e:
                    continue

                with tf:
                    try:
                        meta = json.loads(
                            tf.extractfile(f"{video_id}/metadata.json").read()
                        )
                    except Exception:
                        continue

                    usable_segments = [
                        s for s in meta.get("segments", []) if s.get("status") == "usable"
                    ]
                    segment_map = {
                        (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
                        for seg in usable_segments
                    }

                    members = [m for m in tf.getmembers() if m.name.endswith(".flac")]
                    self.stats.segments_total += len(members)

                    for member in members:
                        parsed = parse_segment_filename(member.name)
                        if not parsed:
                            continue
                        speaker, start_time = parsed
                        seg_meta = segment_map.get((speaker, round(start_time, 2)))
                        if not seg_meta:
                            matches = [
                                s for s in usable_segments
                                if s.get("speaker") == speaker
                                and abs(s.get("start", 0.0) - start_time) < 0.1
                            ]
                            if not matches:
                                continue
                            seg_meta = matches[0]

                        while len(pending) >= max_inflight:
                            drain_completed()

                        segment_bytes = tf.extractfile(member).read()
                        filename_parts = member.name.split("_")
                        try:
                            index = int(filename_parts[-2])
                        except (ValueError, IndexError):
                            index = 0
                        rng_seed = make_segment_seed(self.seed, video_id, speaker, index)

                        future = executor.submit(
                            process_segment,
                            video_id,
                            self.language,
                            segment_bytes,
                            seg_meta,
                            speaker,
                            index,
                            self.chunk_cfg,
                            self.silence_cfg,
                            rng_seed,
                            self.output_format,
                        )
                        future_info[future] = is_val
                        pending.add(future)

                processed_videos += 1
                if self.on_progress:
                    self.on_progress(processed_videos, total_videos)

            while pending:
                drain_completed()

        return self.stats

    def close(self) -> None:
        """Close all writers and manifests."""
        for writer in self.writers.values():
            writer.close()
        self.train_manifest.close()
        self.val_manifest.close()

    def save_stats(self) -> Path:
        """Save processing stats to JSON."""
        stats_dir = self.output_dir / "stats"
        stats_dir.mkdir(parents=True, exist_ok=True)
        stats_path = stats_dir / f"{self.language}_stats_{self.shard_prefix}.json"
        stats_path.write_text(json.dumps(self.stats.to_dict(), indent=2))
        return stats_path


**Confirming task seeding flow**

Planning to review seed_video_tasks to verify how tasks are seeded from Supabase and relate to the language queue for the pipeline summary.


sed -n '1,220p' workers/seed_video_tasks.py


sed -n '1,220p' workers/seed_language_queue.py


sed -n '1,260p' workers/release_stale_claims.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/usr/bin/env python3
"""Seed video_tasks from the existing videos table (Supabase/Postgres)."""

from __future__ import annotations

import argparse
from pathlib import Path

import psycopg
import sys

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))

from prepare_shards import load_env

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Seed video_tasks table.")
    parser.add_argument("--language")
    parser.add_argument(
        "--all-languages",
        action="store_true",
        help="Seed tasks for all languages (ignores --language)",
    )
    parser.add_argument("--limit", type=int, default=None)
    parser.add_argument("--dry-run", action="store_true")
    return parser.parse_args()

def main() -> None:
    args = parse_args()
    env = load_env(Path(".env"))
    db_url = env.get("DATABASE_URL")
    if not db_url:
        raise SystemExit("DATABASE_URL missing in .env")

    with psycopg.connect(db_url) as conn:
        with conn.cursor() as cur:
            if args.all_languages:
                base = """
                    select youtube_id
                    from videos
                    where r2_tar_key is not null
                      and status = 'COMPLETED'
                """
                params = []
            else:
                if not args.language:
                    raise SystemExit("Provide --language or use --all-languages")
                base = """
                    select youtube_id
                    from videos
                    where lower(language) = %s
                      and r2_tar_key is not null
                      and status = 'COMPLETED'
                """
                params = [args.language.lower()]

            if args.limit:
                base += " limit %s"
                params.append(args.limit)

            cur.execute(f"select count(*) from ({base}) as t", params)
            total = cur.fetchone()[0]

    if args.dry_run:
        if args.all_languages:
            print(f"Would insert {total} rows for all languages")
        else:
            print(f"Would insert {total} rows for language={args.language}")
        return

    with psycopg.connect(db_url) as conn:
        with conn.cursor() as cur:
            if args.all_languages:
                query = """
                    insert into video_tasks (video_id, language, r2_tar_key)
                    select youtube_id, lower(language), r2_tar_key
                    from videos
                    where r2_tar_key is not null
                      and status = 'COMPLETED'
                """
                params = []
            else:
                query = """
                    insert into video_tasks (video_id, language, r2_tar_key)
                    select youtube_id, lower(language), r2_tar_key
                    from videos
                    where lower(language) = %s
                      and r2_tar_key is not null
                      and status = 'COMPLETED'
                """
                params = [args.language.lower()]

            if args.limit:
                query += " limit %s"
                params.append(args.limit)

            query += " on conflict (video_id) do nothing"
            cur.execute(query, params)
        conn.commit()

    if args.all_languages:
        print(f"Inserted {total} rows for all languages")
    else:
        print(f"Inserted {total} rows for language={args.language}")

if __name__ == "__main__":
    main()


Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/usr/bin/env python3
"""Seed language_queue from the existing videos table or a provided list."""

from __future__ import annotations

import argparse
import sys
from pathlib import Path

import psycopg

if __package__ in (None, ""):
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from workers.config import EnvConfig

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Seed language_queue table.")
    parser.add_argument(
        "--languages",
        help="Comma-separated language list (overrides auto-discovery)",
    )
    parser.add_argument(
        "--min-count",
        type=int,
        default=1,
        help="Minimum video count to include when auto-discovering languages",
    )
    parser.add_argument("--dry-run", action="store_true")
    return parser.parse_args()

def main() -> None:
    args = parse_args()
    env = EnvConfig.from_file(Path(".env"))
    if not env.database_url:
        raise SystemExit("DATABASE_URL missing in .env")

    with psycopg.connect(env.database_url) as conn:
        with conn.cursor() as cur:
            if args.languages:
                languages = [lang.strip().lower() for lang in args.languages.split(",") if lang.strip()]
                rows = [(lang, 0) for lang in languages]
            else:
                cur.execute(
                    """
                    select lower(language), count(*)
                    from videos
                    where r2_tar_key is not null
                      and status = 'COMPLETED'
                    group by lower(language)
                    having count(*) >= %s
                    order by count(*) desc
                    """,
                    (args.min_count,),
                )
                rows = cur.fetchall()

            if args.dry_run:
                print(f"Would insert {len(rows)} languages into language_queue")
                return

            for language, count in rows:
                cur.execute(
                    """
                    insert into language_queue (language, priority)
                    values (%s, %s)
                    on conflict (language) do nothing
                    """,
                    (language, int(count)),
                )
        conn.commit()

    print(f"Inserted {len(rows)} languages into language_queue")

if __name__ == "__main__":
    main()


Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/usr/bin/env python3
"""Release stale claims from crashed workers.

This script should be run periodically (e.g., via cron) to:
1. Release video_tasks with expired leases back to 'pending'
2. Mark associated shard_runs as 'stale'
3. Log recovery actions for debugging

Usage:
    python release_stale_claims.py [--dry-run] [--env-file .env]
"""

from __future__ import annotations

import argparse
import logging
import sys
from datetime import datetime
from pathlib import Path

import psycopg

if __package__ in (None, ""):
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from workers.config import EnvConfig

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger("stale-recovery")

def release_stale_video_tasks(conn, dry_run: bool = False) -> int:
    """Release video_tasks with expired leases back to pending."""
    with conn.cursor() as cur:
        if dry_run:
            cur.execute(
                """
                select video_id, claimed_by, lease_expires_at, status
                from video_tasks
                where status in ('claimed', 'processing')
                  and lease_expires_at < now()
                """
            )
            stale = cur.fetchall()
            for row in stale:
                logger.info(
                    f"[DRY-RUN] Would release video {row[0]} "
                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
                )
            return len(stale)
        else:
            cur.execute(
                """
                update video_tasks
                set status = 'pending',
                    claimed_by = null,
                    lease_expires_at = null,
                    updated_at = now()
                where status in ('claimed', 'processing')
                  and lease_expires_at < now()
                returning video_id, claimed_by
                """
            )
            released = cur.fetchall()
            for video_id, claimed_by in released:
                logger.info(f"Released video {video_id} (was claimed by {claimed_by})")
            return len(released)

def mark_stale_shards(conn, dry_run: bool = False) -> int:
    """Mark shard_runs with expired leases as stale."""
    with conn.cursor() as cur:
        if dry_run:
            cur.execute(
                """
                select shard_id, claimed_by, lease_expires_at, status
                from shard_runs
                where status in ('claimed', 'processing', 'uploading')
                  and lease_expires_at < now()
                """
            )
            stale = cur.fetchall()
            for row in stale:
                logger.info(
                    f"[DRY-RUN] Would mark shard {row[0]} as stale "
                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
                )
            return len(stale)
        else:
            cur.execute(
                """
                update shard_runs
                set status = 'stale',
                    updated_at = now()
                where status in ('claimed', 'processing', 'uploading')
                  and lease_expires_at < now()
                returning shard_id, claimed_by
                """
            )
            stale = cur.fetchall()
            for shard_id, claimed_by in stale:
                logger.info(f"Marked shard {shard_id} as stale (was claimed by {claimed_by})")
            return len(stale)

def release_stale_languages(conn, dry_run: bool = False) -> int:
    """Release language_queue entries with expired leases back to pending."""
    with conn.cursor() as cur:
        if dry_run:
            cur.execute(
                """
                select language, claimed_by, lease_expires_at, status
                from language_queue
                where status = 'processing'
                  and lease_expires_at < now()
                """
            )
            stale = cur.fetchall()
            for row in stale:
                logger.info(
                    f"[DRY-RUN] Would release language {row[0]} "
                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
                )
            return len(stale)
        else:
            cur.execute(
                """
                update language_queue
                set status = 'pending',
                    claimed_by = null,
                    lease_expires_at = null,
                    updated_at = now()
                where status = 'processing'
                  and lease_expires_at < now()
                returning language, claimed_by
                """
            )
            released = cur.fetchall()
            for language, claimed_by in released:
                logger.info(
                    f"Released language {language} (was claimed by {claimed_by})"
                )
            if released:
                cur.execute(
                    """
                    update language_state
                    set active_language = null,
                        updated_at = now()
                    where id = 1
                      and active_language = any(%s)
                    """,
                    ([row[0] for row in released],),
                )
            return len(released)

def log_recovery_event(conn, video_ids: int, shard_ids: int, language_ids: int) -> None:
    """Log recovery event for audit trail."""
    with conn.cursor() as cur:
        cur.execute(
            """
            insert into video_task_events (event, details)
            values ('stale_recovery', %s)
            """,
            (
                f'{{"videos_released": {video_ids}, '
                f'"shards_marked_stale": {shard_ids}, '
                f'"languages_released": {language_ids}}}',
            ),
        )

def get_stats(conn) -> dict:
    """Get current task statistics."""
    with conn.cursor() as cur:
        cur.execute(
            """
            select status, count(*)
            from video_tasks
            group by status
            """
        )
        video_stats = dict(cur.fetchall())

        cur.execute(
            """
            select status, count(*)
            from shard_runs
            group by status
            """
        )
        shard_stats = dict(cur.fetchall())

        cur.execute(
            """
            select status, count(*)
            from language_queue
            group by status
            """
        )
        language_stats = dict(cur.fetchall())

        cur.execute(
            """
            select count(distinct worker_id)
            from worker_heartbeats_v2
            where last_heartbeat > now() - interval '5 minutes'
            """
        )
        active_workers = cur.fetchone()[0]

    return {
        "video_tasks": video_stats,
        "shard_runs": shard_stats,
        "language_queue": language_stats,
        "active_workers": active_workers,
    }

def main():
    parser = argparse.ArgumentParser(description="Release stale claims from crashed workers")
    parser.add_argument("--dry-run", action="store_true", help="Show what would be done")
    parser.add_argument("--env-file", type=Path, default=Path(".env"))
    parser.add_argument("--stats-only", action="store_true", help="Only show stats, don't recover")
    args = parser.parse_args()

    # Load config
    if args.env_file.exists():
        env_config = EnvConfig.from_file(args.env_file)
    else:
        env_config = EnvConfig.from_env_vars()

    if not env_config.database_url:
        logger.error("DATABASE_URL not configured")
        raise SystemExit(1)

    with psycopg.connect(env_config.database_url) as conn:
        # Show current stats
        stats = get_stats(conn)
        logger.info("=== Current Statistics ===")
        logger.info(f"Video tasks: {stats['video_tasks']}")
        logger.info(f"Shard runs: {stats['shard_runs']}")
        logger.info(f"Active workers (last 5 min): {stats['active_workers']}")

        if args.stats_only:
            return

        logger.info("")
        logger.info("=== Releasing Stale Claims ===")

        # Release stale claims
        videos_released = release_stale_video_tasks(conn, args.dry_run)
        shards_marked = mark_stale_shards(conn, args.dry_run)
        languages_released = release_stale_languages(conn, args.dry_run)

        if not args.dry_run:


Output:
Exit code: 0
Wall time: 0 seconds
Output:
-- Supabase schema for horizontal workers

create table if not exists video_tasks (
  video_id text primary key,
  language text not null,
  r2_tar_key text not null,
  status text not null default 'pending', -- pending|claimed|processing|uploading|completed|failed|skipped
  shard_id uuid,
  claimed_by text,
  lease_expires_at timestamptz,
  attempts int not null default 0,
  last_error text,
  created_at timestamptz default now(),
  updated_at timestamptz default now()
);

create index if not exists video_tasks_language_status_idx
  on video_tasks(language, status);

create table if not exists shard_runs (
  shard_id uuid primary key,
  language text not null,
  target_video_count int not null,
  assigned_video_count int not null,
  status text not null default 'claimed', -- claimed|processing|uploading|completed|failed|stale
  claimed_by text,
  lease_expires_at timestamptz,
  shard_prefix text,
  output_format text not null default 'flac',
  metrics jsonb,
  created_at timestamptz default now(),
  updated_at timestamptz default now()
);

create index if not exists shard_runs_language_status_idx
  on shard_runs(language, status);

create table if not exists worker_heartbeats_v2 (
  worker_id text primary key,
  hostname text,
  instance_type text,
  instance_id text,
  instance_provider text,
  language text,
  status text not null default 'idle', -- idle|claiming|processing|uploading|failed
  shard_id uuid,
  videos_assigned int default 0,
  videos_processed int default 0,
  segments_processed int default 0,
  outputs_generated int default 0,
  last_error text,
  last_heartbeat timestamptz default now()
);

create table if not exists video_task_events (
  id bigserial primary key,
  video_id text,
  shard_id uuid,
  worker_id text,
  event text,
  details jsonb,
  created_at timestamptz default now()
);

create table if not exists language_queue (
  language text primary key,
  priority int not null default 0,
  status text not null default 'pending', -- pending|processing|completed|stale
  claimed_by text,
  lease_expires_at timestamptz,
  created_at timestamptz default now(),
  updated_at timestamptz default now(),
  completed_at timestamptz
);

create table if not exists language_state (
  id int primary key,
  active_language text,
  updated_at timestamptz default now()
);

insert into language_state (id, active_language)
values (1, null)
on conflict (id) do nothing;

alter table worker_heartbeats_v2
  add column if not exists instance_id text;

alter table worker_heartbeats_v2
  add column if not exists instance_provider text;


**Preparing final summary**

I'm consolidating insights from the data preparation pipeline, emphasizing key steps like segmentation, metric calculation, chunking, resampling, and dataset creation into webdataset archives with manifests, while carefully avoiding exposing any secrets from environment files. Next, I'll craft a concise 5-6 sentence summary capturing these stages at a high level.


**Confirming pipeline scope**

I’m verifying the full pipeline scope to cover YouTube data harvesting, segmentation producing per-video tars with metadata, followed by distributed workers generating shards and manifests while interacting with Supabase tasks and R2 buckets. Next, I’ll scan relevant docs or key scripts like prepare_shards.py to confirm missing details before crafting the final 5-6 sentence overview.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
strategy.md
docs/inworld_codec.txt
docs/inworld.pdf
X-Codec-2.0/data_module.py
X-Codec-2.0/criterions/stft_loss.py
X-Codec-2.0/config/train/default.yaml
X-Codec-2.0/criterions/__init__.py
X-Codec-2.0/criterions/gan_loss.py
X-Codec-2.0/config/default.yaml
X-Codec-2.0/criterions/mel_loss.py
X-Codec-2.0/large_scale_save_code.sh
X-Codec-2.0/requirements.txt
X-Codec-2.0/lightning_module.py
X-Codec-2.0/utils.py
X-Codec-2.0/get_tsv.py
X-Codec-2.0/config/dataset/default.yaml
X-Codec-2.0/config/preprocess/default.yaml
X-Codec-2.0/config/model/default.yaml
X-Codec-2.0/inference.py
X-Codec-2.0/README.md
X-Codec-2.0/vq/module.py
X-Codec-2.0/vq/residual_vq.py
X-Codec-2.0/vq/activations.py
X-Codec-2.0/LICENSE
X-Codec-2.0/train.py
X-Codec-2.0/train_slurm.sh
X-Codec-2.0/preprocess.py
X-Codec-2.0/test_audio/output_test/test.flac
X-Codec-2.0/common/__pycache__/audio.cpython-311.pyc
X-Codec-2.0/common/__pycache__/audio.cpython-310.pyc
X-Codec-2.0/common/__pycache__/audio.cpython-39.pyc
X-Codec-2.0/common/__pycache__/schedulers.cpython-311.pyc
X-Codec-2.0/common/__pycache__/schedulers.cpython-312.pyc
X-Codec-2.0/common/__pycache__/schedulers.cpython-310.pyc
X-Codec-2.0/common/__pycache__/audio.cpython-312.pyc
X-Codec-2.0/common/__pycache__/schedulers.cpython-39.pyc
X-Codec-2.0/common/audio.py
X-Codec-2.0/common/schedulers.py
objective.md
Summary.md
X-Codec-2.0/vq/alias_free_torch/resample.py
X-Codec-2.0/vq/alias_free_torch/filter.py
X-Codec-2.0/vq/alias_free_torch/__init__.py
X-Codec-2.0/vq/alias_free_torch/act.py
X-Codec-2.0/vq/factorized_vector_quantize.py
X-Codec-2.0/vq/codec_encoder.py
X-Codec-2.0/vq/__init__.py
X-Codec-2.0/vq/blocks.py
X-Codec-2.0/vq/codec_decoder_vocos.py
X-Codec-2.0/vq/bs_roformer5.py
vastai.md
workers/README.md
workers/docker-compose.yml
workers/worker_plan.md
scripts/fetch_languages.py
workers/validate_r2_uploads.py
scripts/fill_missing_languages.py
scripts/prepare_shards.py
scripts/benchmark_results.json
scripts/supabase_summary.py
scripts/benchmark_gpu.py
X-Codec-2.0/test_audio/input_test/test.flac
workers/worker.py
workers/vast_deploy.py
workers/vast_monitor.py
X-Codec-2.0/module/mpd.py
X-Codec-2.0/module/__init__.py
X-Codec-2.0/module/mstft.py
workers/Dockerfile.gpu
workers/r2_uploader.py
workers/seed_video_tasks.py
workers/release_stale.sql
workers/__init__.py
workers/vast_search.py
workers/storage_manager.py
workers/reset_run.py
workers/vast_bulk_deploy.py
workers/Dockerfile.cpu
workers/entrypoint.sh
workers/seed_language_queue.py
workers/schema.sql
workers/worker_processing.py
workers/release_stale_claims.py
workers/claim_shard.sql
workers/config.py
workers/requirements.txt
vastai_sdk.md
optimisations.md
X-Codec-2.0/inference_save_code.py
scripts/bulk_download.py
scripts/fast_audio_pipeline.py
scripts/PERFORMANCE_ANALYSIS.md
scripts/normalize_languages.py
scripts/__pycache__/gpu_audio_decode.cpython-310.pyc
scripts/__pycache__/optimized_pipeline.cpython-310.pyc
scripts/__pycache__/gpu_audio_metrics_v2.cpython-310.pyc
scripts/__pycache__/fast_audio_pipeline_v2.cpython-310.pyc
scripts/__pycache__/gpu_audio_processor.cpython-310.pyc
scripts/__pycache__/gpu_audio_metrics.cpython-310.pyc
workers/__pycache__/seed_video_tasks.cpython-310.pyc
scripts/__pycache__/prepare_shards.cpython-310.pyc
workers/__pycache__/storage_manager.cpython-310.pyc
scripts/__pycache__/fast_audio_pipeline.cpython-310.pyc
workers/__pycache__/worker_processing.cpython-310.pyc
workers/__pycache__/worker.cpython-310.pyc
workers/__pycache__/config.cpython-310.pyc
workers/__pycache__/seed_language_queue.cpython-310.pyc
workers/__pycache__/r2_uploader.cpython-310.pyc
workers/__pycache__/__init__.cpython-310.pyc
workers/__pycache__/vast_search.cpython-310.pyc
workers/__pycache__/release_stale_claims.cpython-310.pyc
scripts/legacy/README.md
scripts/legacy/benchmark_optimized_pipeline.py
scripts/legacy/gpu_audio_metrics.py
scripts/legacy/gpu_audio_metrics_v2.py
scripts/legacy/clever_pipeline.py
scripts/legacy/gpu_audio_decode.py
scripts/legacy/optimized_pipeline.py
scripts/legacy/fast_audio_pipeline_v2.py
maya3-monitor/package.json
maya3-monitor/postcss.config.mjs
maya3-monitor/tailwind.config.ts
maya3-monitor/tsconfig.json
maya3-monitor/src/app/layout.tsx
maya3-monitor/src/app/page.tsx
maya3-monitor/src/app/globals.css
maya3-monitor/src/app/api/stats/route.ts
maya3-monitor/src/app/api/reboot/route.ts
maya3-monitor/src/lib/supabase.ts
maya3-monitor/src/lib/data.ts
maya3-monitor/src/types/index.ts
maya3-monitor/src/components/ShardTable.tsx
maya3-monitor/src/components/LanguageTable.tsx
maya3-monitor/src/components/WorkerCard.tsx
maya3-monitor/src/components/WorkerStatsTable.tsx
maya3-monitor/src/components/Dashboard.tsx
maya3-monitor/src/components/StatsCard.tsx
maya3-monitor/package-lock.json
maya3-monitor/next.config.mjs
tts/uv.lock
tts/pyproject.toml
tts/CONTRIBUTING.md
tts/Makefile
tts/setup/setup_python.sh
tts/tools/data/data_vectorizer.py
tts/tools/data/data_merger.py
tts/tools/serving/convert_checkpoint.py
tts/tools/serving/inference.py
tts/LICENSE
tts/example/codec/model_config.json
tts/tts/utils/configuration.py
tts/tts/utils/custom_logging.py
tts/example/configs/codec_training_config.json
tts/example/configs/sft.json
tts/example/configs/samples.jsonl
tts/example/configs/rlhf.json
tts/tts/inference/quality_validation.py
tts/tts/inference/inferencing.py
tts/tts/training/codec/codec_datasets.py
tts/tts/training/codec/train_codec.py
tts/tts/training/codec/gan_training_loop.py
tts/tts/training/codec/codec_quality_validation.py
tts/tts/training/evaluation.py
tts/tts/core/codec/criterion.py
tts/tts/core/codec/decoder_modules.py
tts/tts/core/codec/decoder.py
tts/tts/training/main.py
tts/tts/core/codec/encoding.py
tts/tts/core/codec/activations.py
tts/tts/core/codec/encoder_modules.py
tts/tts/training/training_loop.py
tts/tts/core/codec/discriminator.py
tts/tts/core/codec/encoder.py
tts/tts/core/prompting.py
tts/tts/core/codec/upsampler.py
tts/tts/core/constants.py
tts/tts/core/codec/decoding.py
tts/tts/core/tokenization.py
tts/tts/core/codec/filters.py
tts/tts/core/modeling.py
tts/tts/core/lora.py
tts/tts/core/optimization.py
tts/tts/training/checkpointing.py
tts/tts/training/environment.py
tts/README.md
tts/tts/__init__.py
tts/tts/training/rlhf/ecapa_tdnn.py
tts/tts/training/rlhf/rlhf_main.py
tts/tts/training/rlhf/reward_utils.py
tts/tts/training/rlhf/rewards.py
tts/tts/training/rlhf/run_rlhf_combine.sh
tts/tts/data/data_utils.py
tts/tts/data/text_normalization.py
tts/tts/data/caching.py
tts/tts/data/tts_datasets.py
tts/tts/data/data_sample.py
tts/tts/data/filtering.py
tts/tts/data/datasets/pretraining.py
tts/tts/data/datasets/rlhf.py
tts/tts/data/datasets/finetuning.py


**Checking segmentation source in pipeline**

I'm verifying whether segmentation metadata in the tar files originates from prepare_shards or the GPU-based segmentation in fast_audio_pipeline to clarify the pipeline summary.


sed -n '1,240p' scripts/prepare_shards.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

import argparse
import io
import json
import math
import os
import random
import tarfile
import threading
import time
import zlib
from collections import defaultdict
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Optional

import boto3
import numpy as np
import psycopg
import soundfile as sf
from boto3.s3.transfer import TransferConfig
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
from scipy import signal
from tqdm import tqdm

try:
    import pyloudnorm as pyln
except Exception:  # pragma: no cover - optional dependency
    pyln = None

# GPU pipeline (optional)
try:
    from fast_audio_pipeline import EquivalentBatchProcessor, SegmentInput, OutputSegment
    HAS_GPU_PIPELINE = True
except ImportError:
    HAS_GPU_PIPELINE = False
    EquivalentBatchProcessor = None
    SegmentInput = None
    OutputSegment = None

ENV_PATH = Path(".env")
LANGUAGES = [
    "telugu",
    "malayalam",
    "english",
    "hindi",
    "punjabi",
    "tamil",
    "kannada",
    "gujarati",
    "bengali",
    "odia",
    "marathi",
    "assamese",
]

# Global timing stats (thread-safe)
_timing_lock = threading.Lock()
_timing_stats: dict[str, list[float]] = defaultdict(list)
_timing_enabled = False

@contextmanager
def timer(name: str):
    """Context manager for timing code blocks."""
    if not _timing_enabled:
        yield
        return
    start = time.perf_counter()
    try:
        yield
    finally:
        elapsed = time.perf_counter() - start
        with _timing_lock:
            _timing_stats[name].append(elapsed)

def print_timing_stats():
    """Print aggregated timing statistics."""
    if not _timing_stats:
        return
    print("\n=== Timing Statistics ===")
    total_time = sum(sum(times) for times in _timing_stats.values())
    for name, times in sorted(_timing_stats.items()):
        count = len(times)
        total = sum(times)
        avg = total / count if count > 0 else 0
        percent = (total / total_time * 100) if total_time > 0 else 0
        print(f"{name:35s}: {count:6d} calls, {total:8.2f}s total, {avg*1000:8.2f}ms avg, {percent:5.1f}%")
    print(f"{'TOTAL':35s}:              {total_time:8.2f}s")
    print("=" * 80)

_window_cache: dict[int, np.ndarray] = {}
_freq_cache: dict[tuple[int, int], np.ndarray] = {}
_cache_lock = threading.Lock()
_lufs_local = threading.local()

def get_hann_window(length: int) -> np.ndarray:
    window = _window_cache.get(length)
    if window is not None:
        return window
    with _cache_lock:
        window = _window_cache.get(length)
        if window is None:
            window = np.hanning(length)
            _window_cache[length] = window
    return window

def get_rfftfreq(length: int, sr: int) -> np.ndarray:
    key = (length, sr)
    freqs = _freq_cache.get(key)
    if freqs is not None:
        return freqs
    with _cache_lock:
        freqs = _freq_cache.get(key)
        if freqs is None:
            freqs = np.fft.rfftfreq(length, d=1.0 / sr)
            _freq_cache[key] = freqs
    return freqs

def get_lufs_meter(sr: int) -> Optional[Any]:
    if pyln is None:
        return None
    meters = getattr(_lufs_local, "meters", None)
    if meters is None:
        meters = {}
        _lufs_local.meters = meters
    meter = meters.get(sr)
    if meter is None:
        meter = pyln.Meter(sr)
        meters[sr] = meter
    return meter

@dataclass(frozen=True)
class VideoRecord:
    youtube_id: str
    r2_tar_key: str

@dataclass(frozen=True)
class SegmentOutput:
    sample_rate: int
    audio_bytes: bytes
    metadata: dict[str, Any]

@dataclass(frozen=True)
class ChunkConfig:
    enabled: bool
    min_sec: float
    max_sec: float
    threshold_sec: float
    all_buckets: bool

@dataclass(frozen=True)
class SilenceConfig:
    enabled: bool
    frame_sec: float
    hop_sec: float
    search_sec: float
    percentile: float
    db_drop: float

def load_env(env_path: Path) -> dict[str, str]:
    env: dict[str, str] = {}
    for line in env_path.read_text().splitlines():
        stripped = line.strip()
        if not stripped or stripped.startswith("#") or "=" not in stripped:
            continue
        if stripped.startswith("export "):
            stripped = stripped[len("export ") :]
        key, value = stripped.split("=", 1)
        env[key.strip()] = value.strip()
    return env

def make_r2_client(env: dict[str, str]):
    endpoint = env.get("R2_ENDPOINT_URL")
    access_key = env.get("R2_ACCESS_KEY_ID")
    secret_key = env.get("R2_SECRET_ACCESS_KEY")
    if not endpoint or not access_key or not secret_key:
        raise SystemExit("Missing R2 credentials in .env")
    return boto3.client(
        "s3",
        endpoint_url=endpoint,
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
    )

def resolve_r2_key(
    s3, bucket: str, r2_key: str, prefix: Optional[str]
) -> Optional[str]:
    if not r2_key:
        return None
    key = r2_key.lstrip("/")
    candidates = [key]

    if prefix:
        cleaned_prefix = prefix.strip("/")
        if not key.startswith(cleaned_prefix):
            candidates.append(f"{cleaned_prefix}/{key}")

    base = os.path.basename(key)
    if base != key:
        candidates.append(base)
        if prefix:
            candidates.append(f"{prefix.strip('/')}/{base}")

    for candidate in candidates:
        try:
            with timer("r2_head_object"):
                s3.head_object(Bucket=bucket, Key=candidate)
            return candidate
        except Exception:
            continue
    return None

def parse_source_url(source_url: Optional[str], default_bucket: Optional[str]) -> Optional[tuple[str, str]]:
    if not source_url or not default_bucket:
        return None
    cleaned = source_url.lstrip("/")
    if cleaned.startswith(default_bucket + "/"):
        return default_bucket, cleaned[len(default_bucket) + 1 :]
    return default_bucket, cleaned

def fetch_video_records(
    db_url: str,
    language: str,


**Checking early audio pipeline role**

I'm planning to review the top of the fast_audio_pipeline.py script to understand if it handles pre-segmentation from raw video, as this could clarify how segmentation fits into the overall worker pipeline.


sed -n '1,200p' scripts/fast_audio_pipeline.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""GPU-accelerated audio processing - semantically equivalent to CPU path.

This module provides a drop-in replacement for segment processing that:
1. Uses EXACT same chunking logic (make_chunk_windows)
2. Uses EXACT same metrics computation (compute_audio_metrics with pyloudnorm)
3. Only accelerates I/O-bound operations:
   - Parallel FLAC decoding
   - GPU-batched resampling
   - Parallel WAV encoding

The GPU path produces IDENTICAL outputs to the CPU path.
"""

from __future__ import annotations

import io
import math
import random
import threading
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import Any, Optional

import numpy as np
import soundfile as sf

# Import exact same functions from prepare_shards for equivalence
from prepare_shards import (
    ChunkConfig,
    SilenceConfig,
    SegmentOutput,
    make_chunk_windows,
    compute_audio_metrics,
    compute_quality_score,
    duration_bucket,
    classify_native_bucket,
)

# GPU acceleration (optional)
try:
    import torch
    import torchaudio
    HAS_TORCH = True
except ImportError:
    HAS_TORCH = False
    torch = None
    torchaudio = None

@dataclass
class SegmentInput:
    """Input segment for processing."""
    video_id: str
    language: str
    segment_bytes: bytes
    segment_meta: dict[str, Any]
    speaker: str
    index: int
    is_val: bool
    rng_seed: int

@dataclass
class OutputSegment:
    """Final output ready for shard writing."""
    sample_rate: int
    audio_bytes: bytes
    metadata: dict[str, Any]
    is_val: bool

class GPUResampler:
    """GPU-accelerated resampler using torchaudio."""

    def __init__(self, device: str = "cuda"):
        self.use_gpu = HAS_TORCH and torch.cuda.is_available()
        self.device = torch.device(device if self.use_gpu else "cpu")
        self._resamplers: dict[tuple[int, int], Any] = {}
        self._lock = threading.Lock()

        if self.use_gpu:
            print(f"GPUResampler: Using {torch.cuda.get_device_name()}")

    def resample(self, audio: np.ndarray, src_sr: int, dst_sr: int) -> np.ndarray:
        """Resample audio using GPU if available."""
        if src_sr == dst_sr:
            return audio

        if not self.use_gpu:
            # CPU fallback using scipy
            from scipy import signal
            gcd = math.gcd(src_sr, dst_sr)
            up = dst_sr // gcd
            down = src_sr // gcd
            return signal.resample_poly(audio, up, down).astype(np.float32, copy=False)

        # GPU path
        resampler = self._get_resampler(src_sr, dst_sr)
        audio_t = torch.from_numpy(audio).unsqueeze(0).to(self.device)
        with torch.no_grad():
            resampled_t = resampler(audio_t)
        return resampled_t.squeeze(0).cpu().numpy()

    def resample_batch(
        self, audios: list[np.ndarray], src_sr: int, dst_sr: int
    ) -> list[np.ndarray]:
        """Resample multiple audio arrays to the same target rate."""
        if src_sr == dst_sr:
            return audios

        if not self.use_gpu or not audios:
            return [self.resample(a, src_sr, dst_sr) for a in audios]

        # Batch on GPU
        resampler = self._get_resampler(src_sr, dst_sr)

        # Pad to max length for batching
        lengths = [a.shape[0] for a in audios]
        max_len = max(lengths)
        batch = np.zeros((len(audios), max_len), dtype=np.float32)
        for i, a in enumerate(audios):
            batch[i, :a.shape[0]] = a

        batch_t = torch.from_numpy(batch).to(self.device)
        with torch.no_grad():
            resampled_t = resampler(batch_t)
        resampled_batch = resampled_t.cpu().numpy()

        # Trim to actual lengths
        ratio = dst_sr / src_sr
        results = []
        for i, orig_len in enumerate(lengths):
            new_len = int(orig_len * ratio)
            results.append(resampled_batch[i, :new_len])

        return results

    def _get_resampler(self, src_sr: int, dst_sr: int):
        """Get or create a resampler for the given rate pair."""
        key = (src_sr, dst_sr)
        if key not in self._resamplers:
            with self._lock:
                if key not in self._resamplers:
                    self._resamplers[key] = torchaudio.transforms.Resample(
                        src_sr, dst_sr, dtype=torch.float32
                    ).to(self.device)
        return self._resamplers[key]

class EquivalentBatchProcessor:
    """Batch processor that produces IDENTICAL outputs to CPU path.

    This processor:
    1. Decodes segments in parallel (CPU ThreadPool)
    2. Applies EXACT same chunking logic as CPU path
    3. Computes metrics using EXACT same functions (including pyloudnorm LUFS)
    4. Resamples using GPU (same algorithm, just faster)
    5. Encodes in parallel (CPU ThreadPool)

    The only difference from CPU is parallelism/hardware, not algorithms.
    """

    def __init__(
        self,
        decode_workers: int = 8,
        encode_workers: int = 8,
        output_format: str = "wav",
        chunk_cfg: Optional[ChunkConfig] = None,
        silence_cfg: Optional[SilenceConfig] = None,
    ):
        self.decode_workers = decode_workers
        self.encode_workers = encode_workers
        self.output_format = output_format
        self.chunk_cfg = chunk_cfg or ChunkConfig(
            enabled=True, min_sec=6.0, max_sec=10.0, threshold_sec=12.0, all_buckets=False
        )
        self.silence_cfg = silence_cfg or SilenceConfig(
            enabled=True, frame_sec=0.05, hop_sec=0.01,
            search_sec=0.5, percentile=20.0, db_drop=20.0
        )

        self.resampler = GPUResampler()

        self.stats = {
            "decoded": 0,
            "chunks_emitted": 0,
            "segments_chunked": 0,
            "dropped_too_short": 0,
            "dropped_too_long": 0,
            "dropped_clipped": 0,
        }

    def process_segments(
        self, segments: list[SegmentInput]
    ) -> list[OutputSegment]:
        """Process segments and return outputs - equivalent to CPU path.

        This method mirrors the exact logic of process_segment() in prepare_shards.py.
        """
        if not segments:


**Searching for segmentation pipeline**

I'm looking for the segmentation pipeline by searching for "pyannote" references, since fast_audio_pipeline handles audio processing but not segmentation. I'll check scripts/optimized_pipeline next.


rg -n "pyannote|diar|speaker|whisper|vad|pann|music" scripts workers | head -n 200


Output:
Exit code: 0
Wall time: 0 seconds
Output:
scripts/prepare_shards.py:352:def compute_quality_score(metrics: dict[str, float], has_music: bool) -> float:
scripts/prepare_shards.py:356:        + 0.2 * (1.0 if not has_music else 0.5)
scripts/prepare_shards.py:523:def make_segment_seed(base_seed: int, video_id: str, speaker: str, index: int) -> int:
scripts/prepare_shards.py:524:    key = f"{video_id}-{speaker}-{index}-{base_seed}"
scripts/prepare_shards.py:529:    """Parse segment filename to extract speaker and start time.
scripts/prepare_shards.py:534:    Returns (speaker, start_time) for matching against metadata.
scripts/prepare_shards.py:549:    speaker = "_".join(parts[:-2])
scripts/prepare_shards.py:562:    return speaker, start_time
scripts/prepare_shards.py:570:    speaker: str,
scripts/prepare_shards.py:599:        music_stats = segment_meta.get("music_stats") or {}
scripts/prepare_shards.py:600:        has_music = bool(music_stats.get("has_music", False))
scripts/prepare_shards.py:627:            sampling_weight = base_weight * (0.5 if has_music else 1.0)
scripts/prepare_shards.py:628:            quality_score = compute_quality_score(metrics, has_music)
scripts/prepare_shards.py:653:                "music_prob": float(music_stats.get("music_mean", 0.0)),
scripts/prepare_shards.py:654:                "music_stats": music_stats,
scripts/prepare_shards.py:655:                "speaker": speaker,
scripts/prepare_shards.py:657:                "vad_speech_regions": [[0.0, slice_duration]],
scripts/prepare_shards.py:674:        sample_id_base = f"{video_id}__{speaker}_{index:04d}"
scripts/prepare_shards.py:1375:                        (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
scripts/prepare_shards.py:1387:                    speaker, start_time = parsed
scripts/prepare_shards.py:1388:                    seg_meta = segment_map.get((speaker, round(start_time, 2)))
scripts/prepare_shards.py:1392:                            if s.get("speaker") == speaker
scripts/prepare_shards.py:1406:                    rng_seed = make_segment_seed(args.seed, video_id, speaker, index)
scripts/prepare_shards.py:1413:                        speaker=speaker,
scripts/prepare_shards.py:1452:                    # Create lookup by (speaker, start) for robust matching
scripts/prepare_shards.py:1457:                            (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
scripts/prepare_shards.py:1469:                        speaker, start_time = parsed
scripts/prepare_shards.py:1470:                        # Match by (speaker, start) with rounding for float precision
scripts/prepare_shards.py:1471:                        seg_meta = segment_map.get((speaker, round(start_time, 2)))
scripts/prepare_shards.py:1476:                                if s.get("speaker") == speaker
scripts/prepare_shards.py:1494:                        rng_seed = make_segment_seed(args.seed, video_id, speaker, index)
scripts/prepare_shards.py:1502:                            speaker,
scripts/benchmark_gpu.py:76:                (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
scripts/benchmark_gpu.py:86:                speaker, start_time = parsed
scripts/benchmark_gpu.py:87:                seg_meta = segment_map.get((speaker, round(start_time, 2)))
scripts/benchmark_gpu.py:91:                        if s.get("speaker") == speaker
scripts/benchmark_gpu.py:104:                rng_seed = make_segment_seed(seed, video_id, speaker, index)
scripts/benchmark_gpu.py:111:                    speaker=speaker,
scripts/benchmark_gpu.py:147:            speaker="SPEAKER_00",
workers/worker_processing.py:227:def compute_quality_score(metrics: dict[str, float], has_music: bool) -> float:
workers/worker_processing.py:232:        + 0.2 * (1.0 if not has_music else 0.5)
workers/worker_processing.py:399:def make_segment_seed(base_seed: int, video_id: str, speaker: str, index: int) -> int:
workers/worker_processing.py:401:    key = f"{video_id}-{speaker}-{index}-{base_seed}"
workers/worker_processing.py:406:    """Parse segment filename to extract speaker and start time."""
workers/worker_processing.py:416:    speaker = "_".join(parts[:-2])
workers/worker_processing.py:427:    return speaker, start_time
workers/worker_processing.py:435:    speaker: str,
workers/worker_processing.py:463:    music_stats = segment_meta.get("music_stats") or {}
workers/worker_processing.py:464:    has_music = bool(music_stats.get("has_music", False))
workers/worker_processing.py:491:        sampling_weight = base_weight * (0.5 if has_music else 1.0)
workers/worker_processing.py:492:        quality_score = compute_quality_score(metrics, has_music)
workers/worker_processing.py:517:            "music_prob": float(music_stats.get("music_mean", 0.0)),
workers/worker_processing.py:518:            "music_stats": music_stats,
workers/worker_processing.py:519:            "speaker": speaker,
workers/worker_processing.py:520:            "vad_speech_regions": [[0.0, slice_duration]],
workers/worker_processing.py:537:    sample_id_base = f"{video_id}__{speaker}_{index:04d}"
workers/worker_processing.py:867:                        (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
workers/worker_processing.py:878:                        speaker, start_time = parsed
workers/worker_processing.py:879:                        seg_meta = segment_map.get((speaker, round(start_time, 2)))
workers/worker_processing.py:883:                                if s.get("speaker") == speaker
workers/worker_processing.py:899:                        rng_seed = make_segment_seed(self.seed, video_id, speaker, index)
workers/worker_processing.py:907:                            speaker,
scripts/legacy/fast_audio_pipeline_v2.py:59:    speaker: str
scripts/legacy/fast_audio_pipeline_v2.py:202:        music_stats = seg.segment_meta.get("music_stats") or {}
scripts/legacy/fast_audio_pipeline_v2.py:203:        has_music = bool(music_stats.get("has_music", False))
scripts/legacy/fast_audio_pipeline_v2.py:205:        sample_id_base = f"{seg.video_id}__{seg.speaker}_{seg.index:04d}"
scripts/legacy/fast_audio_pipeline_v2.py:240:                chunk_original_end, [16000], has_music, music_stats, seg.speaker, seg.is_val
scripts/legacy/fast_audio_pipeline_v2.py:292:                        chunk_original_end, slice_targets, has_music, music_stats, seg.speaker, seg.is_val
scripts/legacy/fast_audio_pipeline_v2.py:299:                    targets_high, has_music, music_stats, seg.speaker, seg.is_val
scripts/legacy/fast_audio_pipeline_v2.py:308:        has_music, music_stats, speaker, is_val
scripts/legacy/fast_audio_pipeline_v2.py:328:        sampling_weight = base_weight * (0.5 if has_music else 1.0)
scripts/legacy/fast_audio_pipeline_v2.py:329:        quality_score = compute_quality_score(metrics, has_music)
scripts/legacy/fast_audio_pipeline_v2.py:355:            "music_prob": float(music_stats.get("music_mean", 0.0)),
scripts/legacy/fast_audio_pipeline_v2.py:356:            "music_stats": music_stats,
scripts/legacy/fast_audio_pipeline_v2.py:357:            "speaker": speaker,
scripts/legacy/fast_audio_pipeline_v2.py:358:            "vad_speech_regions": [[0.0, slice_duration]],
scripts/legacy/fast_audio_pipeline_v2.py:420:            speaker="SPEAKER_00",
scripts/legacy/benchmark_optimized_pipeline.py:97:            (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
scripts/legacy/benchmark_optimized_pipeline.py:106:            speaker, start_time = parsed
scripts/legacy/benchmark_optimized_pipeline.py:107:            seg_meta = segment_map.get((speaker, round(start_time, 2)))
scripts/legacy/benchmark_optimized_pipeline.py:111:                    if s.get("speaker") == speaker
scripts/legacy/benchmark_optimized_pipeline.py:124:            rng_seed = make_segment_seed(seed, video_id, speaker, index)
scripts/legacy/benchmark_optimized_pipeline.py:131:                speaker=speaker,
scripts/legacy/benchmark_optimized_pipeline.py:382:                segment.speaker,
scripts/fast_audio_pipeline.py:57:    speaker: str
scripts/fast_audio_pipeline.py:250:        music_stats = seg.segment_meta.get("music_stats") or {}
scripts/fast_audio_pipeline.py:251:        has_music = bool(music_stats.get("has_music", False))
scripts/fast_audio_pipeline.py:253:        sample_id_base = f"{seg.video_id}__{seg.speaker}_{seg.index:04d}"
scripts/fast_audio_pipeline.py:306:                has_music=has_music,
scripts/fast_audio_pipeline.py:307:                music_stats=music_stats,
scripts/fast_audio_pipeline.py:308:                speaker=seg.speaker,
scripts/fast_audio_pipeline.py:379:                        has_music=has_music,
scripts/fast_audio_pipeline.py:380:                        music_stats=music_stats,
scripts/fast_audio_pipeline.py:381:                        speaker=seg.speaker,
scripts/fast_audio_pipeline.py:401:                    has_music=has_music,
scripts/fast_audio_pipeline.py:402:                    music_stats=music_stats,
scripts/fast_audio_pipeline.py:403:                    speaker=seg.speaker,
scripts/fast_audio_pipeline.py:425:        has_music: bool,
scripts/fast_audio_pipeline.py:426:        music_stats: dict,
scripts/fast_audio_pipeline.py:427:        speaker: str,
scripts/fast_audio_pipeline.py:447:        sampling_weight = base_weight * (0.5 if has_music else 1.0)
scripts/fast_audio_pipeline.py:448:        quality_score = compute_quality_score(metrics, has_music)
scripts/fast_audio_pipeline.py:474:            "music_prob": float(music_stats.get("music_mean", 0.0)),
scripts/fast_audio_pipeline.py:475:            "music_stats": music_stats,
scripts/fast_audio_pipeline.py:476:            "speaker": speaker,
scripts/fast_audio_pipeline.py:477:            "vad_speech_regions": [[0.0, slice_duration]],
scripts/fast_audio_pipeline.py:546:            speaker="SPEAKER_00",
scripts/legacy/optimized_pipeline.py:101:    speaker: str
scripts/legacy/optimized_pipeline.py:129:    has_music: bool = False
scripts/legacy/optimized_pipeline.py:489:            music_stats = seg.segment_meta.get("music_stats") or {}
scripts/legacy/optimized_pipeline.py:490:            has_music = bool(music_stats.get("has_music", False))
scripts/legacy/optimized_pipeline.py:492:            sample_id_base = f"{seg.video_id}__{seg.speaker}_{seg.index:04d}"
scripts/legacy/optimized_pipeline.py:519:                sampling_weight = base_weight * (0.5 if has_music else 1.0)
scripts/legacy/optimized_pipeline.py:544:                    "speaker": seg.speaker,
scripts/legacy/optimized_pipeline.py:545:                    "music_prob": float(music_stats.get("music_mean", 0.0)),
scripts/legacy/optimized_pipeline.py:546:                    "music_stats": music_stats,
scripts/legacy/optimized_pipeline.py:547:                    "vad_speech_regions": [[0.0, slice_duration]],
scripts/legacy/optimized_pipeline.py:564:                    has_music=has_music,
scripts/legacy/optimized_pipeline.py:579:                    sampling_weight = base_weight * (0.5 if has_music else 1.0)
scripts/legacy/optimized_pipeline.py:592:                        "speaker": seg.speaker,
scripts/legacy/optimized_pipeline.py:593:                        "music_prob": float(music_stats.get("music_mean", 0.0)),
scripts/legacy/optimized_pipeline.py:594:                        "music_stats": music_stats,
scripts/legacy/optimized_pipeline.py:595:                        "vad_speech_regions": [[0.0, full_duration]],
scripts/legacy/optimized_pipeline.py:611:                        has_music=has_music,
scripts/legacy/optimized_pipeline.py:655:                    # Compute quality score (use music_stats boolean to match CPU)
scripts/legacy/optimized_pipeline.py:656:                    quality_score = self._compute_quality_score(metrics, chunk.has_music)
scripts/legacy/optimized_pipeline.py:659:    def _compute_quality_score(self, metrics: dict, has_music: bool) -> float:
scripts/legacy/optimized_pipeline.py:669:            + 0.2 * (1.0 if not has_music else 0.5)
scripts/legacy/clever_pipeline.py:60:    speaker: str
scripts/legacy/clever_pipeline.py:293:            music_stats = seg.segment_meta.get("music_stats") or {}
scripts/legacy/clever_pipeline.py:294:            has_music = bool(music_stats.get("has_music", False))
scripts/legacy/clever_pipeline.py:296:            sample_id_base = f"{seg.video_id}__{seg.speaker}_{seg.index:04d}"
scripts/legacy/clever_pipeline.py:313:                sampling_weight = base_weight * (0.5 if has_music else 1.0)
scripts/legacy/clever_pipeline.py:345:                            "speaker": seg.speaker,
scripts/legacy/clever_pipeline.py:371:                fb_sampling_weight = fb_weight * (0.5 if has_music else 1.0)
scripts/legacy/clever_pipeline.py:388:                        "speaker": seg.speaker,


**Summarizing full segmentation pipeline**

The pipeline starts with YouTube audio stored in R2, followed by a GPU-based single-speaker segmentation producing per-video tarballs plus metadata stored in R2 "cleaned" bucket. Distributed shard workers claim tasks from Supabase, process segments with chunking, silence detection, metrics, and filtering, resample audio into multiple sample rates, and write webdataset shards and manifests. Results upload back to R2, and task progress is managed with deterministic shard IDs, leases, skip-locked queries, and heartbeats ensuring safety and recovery. Prior steps include storing .webm files in R2 after proxy downloads and Gemini classification helping prep data.


**Refining pipeline stages based on R2 buckets**

The raw `.webm` files reside in the R2 "test" bucket, and GPU workers run segmentation to produce cleaned `.tar` and FLAC data in "1-cleaned-data". The existing worker code processes from cleaned tars to shards, so the raw `.webm` to cleaned segmentation likely occurs elsewhere, possibly in scripts like `bulk_download.py` or other pipeline helpers. Next, checking `bulk_download.py` to clarify its role in downloading or processing data.


sed -n '1,200p' scripts/bulk_download.py