Output:
Exit code: 0
Wall time: 0 seconds
Output:
                    float(segment_start) + offset_sec if segment_start is not None else None
                )
                chunk_end = (
                    chunk_start + slice_duration if chunk_start is not None else None
                )
                chunk_original_start = (
                    float(original_start) + offset_sec if original_start is not None else None
                )
                chunk_original_end = (
                    chunk_original_start + slice_duration
                    if chunk_original_start is not None
                    else None
                )

                sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
                emit_outputs(
                    audio_slice,
                    slice_metrics,
                    sample_id,
                    index,
                    chunk_idx,
                    chunk_start,
                    chunk_end,
                    chunk_original_start,
                    chunk_original_end,
                    slice_targets,
                )
        else:
            emit_outputs(
                audio,
                full_metrics,
                sample_id_base,
                index,
                None,
                segment_start,
                segment_end,
                original_start,
                original_end,
                targets_high,
            )

    return outputs, stats

class ShardWriter:
    """Write samples to tar + jsonl shards."""

    def __init__(
        self, output_dir: Path, prefix: str, target_size_bytes: int, audio_ext: str
    ) -> None:
        self.output_dir = output_dir
        self.prefix = prefix
        self.target_size_bytes = target_size_bytes
        self.audio_ext = audio_ext
        self.shard_index = 0
        self.current_tar: Optional[tarfile.TarFile] = None
        self.current_jsonl = None
        self.current_size = 0
        self.current_count = 0
        self.current_name = ""
        self.output_dir.mkdir(parents=True, exist_ok=True)

    def _open_new_shard(self) -> None:
        if self.current_tar:
            self.close()
        shard_name = f"{self.prefix}-{self.shard_index:06d}"
        tar_path = self.output_dir / f"{shard_name}.tar"
        jsonl_path = self.output_dir / f"{shard_name}.jsonl"
        self.current_tar = tarfile.open(tar_path, "w")
        self.current_jsonl = jsonl_path.open("w", encoding="utf-8")
        self.current_size = 0
        self.current_count = 0
        self.current_name = shard_name
        self.shard_index += 1

    def add_sample(self, sample_id: str, audio_bytes: bytes, metadata: dict[str, Any]) -> str:
        if self.current_tar is None:
            self._open_new_shard()
        if (
            self.current_count > 0
            and self.current_size + len(audio_bytes) > self.target_size_bytes
        ):
            self._open_new_shard()

        assert self.current_tar is not None
        assert self.current_jsonl is not None

        arcname = f"{sample_id}.{self.audio_ext}"
        info = tarfile.TarInfo(name=arcname)
        info.size = len(audio_bytes)
        info.mtime = int(time.time())
        self.current_tar.addfile(info, io.BytesIO(audio_bytes))
        self.current_jsonl.write(json.dumps(metadata) + "\n")

        self.current_size += len(audio_bytes)
        self.current_count += 1
        return self.current_name

    def close(self) -> None:
        if self.current_tar:
            self.current_tar.close()
        if self.current_jsonl:
            self.current_jsonl.close()
        self.current_tar = None
        self.current_jsonl = None

class VideoProcessor:
    """Process videos and write to shards."""

    def __init__(
        self,
        output_dir: Path,
        language: str,
        shard_prefix: str,
        chunk_cfg: ChunkConfig,
        silence_cfg: SilenceConfig,
        output_format: str = "flac",
        target_shard_size_mb: int = 5120,
        segment_workers: int = 8,
        seed: int = 17,
        val_ratio: float = 0.01,
        on_progress: Optional[Callable[[int, int], None]] = None,
    ):
        self.output_dir = output_dir
        self.language = language
        self.shard_prefix = shard_prefix
        self.chunk_cfg = chunk_cfg
        self.silence_cfg = silence_cfg
        self.output_format = output_format
        self.seed = seed
        self.val_ratio = val_ratio
        self.segment_workers = segment_workers
        self.on_progress = on_progress

        audio_ext = "wav" if output_format == "wav" else "flac"
        target_bytes = target_shard_size_mb * 1024 * 1024

        self.writers = {
            16000: ShardWriter(
                output_dir / f"{language}_16", shard_prefix, target_bytes, audio_ext
            ),
            24000: ShardWriter(
                output_dir / f"{language}_24", shard_prefix, target_bytes, audio_ext
            ),
            32000: ShardWriter(
                output_dir / f"{language}_32", shard_prefix, target_bytes, audio_ext
            ),
            44100: ShardWriter(
                output_dir / f"{language}_44", shard_prefix, target_bytes, audio_ext
            ),
        }

        # Manifests
        manifest_dir = output_dir / "manifests"
        manifest_dir.mkdir(parents=True, exist_ok=True)
        self.train_manifest = (manifest_dir / f"{language}_train_{shard_prefix}.jsonl").open(
            "w", encoding="utf-8"
        )
        self.val_manifest = (manifest_dir / f"{language}_val_{shard_prefix}.jsonl").open(
            "w", encoding="utf-8"
        )

        self.stats = ProcessingStats()
        self.audio_ext = audio_ext

    def process_videos(
        self,
        download_paths: dict[str, Path],
        val_ids: Optional[set[str]] = None,
    ) -> ProcessingStats:
        """Process all videos and write to shards."""
        if val_ids is None:
            # Determine val set
            video_ids = list(download_paths.keys())
            random.seed(self.seed)
            val_count = max(1, int(len(video_ids) * self.val_ratio)) if len(video_ids) > 1 else 0
            val_ids = set(random.sample(video_ids, val_count)) if val_count > 0 else set()

        max_inflight = max(128, self.segment_workers * 2)
        pending: set[Any] = set()
        future_info: dict[Any, bool] = {}

        def handle_future(future) -> None:
            is_val = future_info.pop(future, False)
            outputs, seg_stats = future.result()
            self.stats.dropped_too_short += seg_stats["dropped_too_short"]
            self.stats.dropped_too_long += seg_stats["dropped_too_long"]
            self.stats.dropped_clipped += seg_stats["dropped_clipped"]
            self.stats.segments_chunked += seg_stats.get("segments_chunked", 0)
            self.stats.chunks_emitted += seg_stats.get("chunks_emitted", 0)

            for out in outputs:
                self.stats.segments_kept[out.sample_rate] += 1
                if out.sample_rate == 16000:
                    bucket = out.metadata.get("native_sr_bucket")
                    if bucket:
                        self.stats.native_sr_buckets[bucket] = (
                            self.stats.native_sr_buckets.get(bucket, 0) + 1
                        )
                writer = self.writers[out.sample_rate]
                shard_name = writer.add_sample(
                    out.metadata["id"], out.audio_bytes, out.metadata
                )
                manifest_entry = dict(out.metadata)
                manifest_entry["shard"] = f"{shard_name}.tar"
                manifest_entry["audio"] = f"{out.metadata['id']}.{self.audio_ext}"
                line = json.dumps(manifest_entry) + "\n"
                if is_val:
                    self.val_manifest.write(line)
                else:
                    self.train_manifest.write(line)

        def drain_completed() -> None:
            nonlocal pending
            if not pending:
                return
            done, pending = wait(pending, return_when=FIRST_COMPLETED)
            for future in done:
                handle_future(future)

        total_videos = len(download_paths)
        processed_videos = 0

        with ThreadPoolExecutor(max_workers=self.segment_workers) as executor:
            for video_id, tar_path in download_paths.items():
                self.stats.videos += 1
                is_val = video_id in val_ids

                try:
                    tf = tarfile.open(tar_path, "r")
                except Exception as e:
                    continue

                with tf:
                    try:
                        meta = json.loads(
                            tf.extractfile(f"{video_id}/metadata.json").read()
                        )
                    except Exception:
                        continue

                    usable_segments = [
                        s for s in meta.get("segments", []) if s.get("status") == "usable"
                    ]
                    segment_map = {
                        (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
                        for seg in usable_segments
                    }

                    members = [m for m in tf.getmembers() if m.name.endswith(".flac")]
                    self.stats.segments_total += len(members)

                    for member in members:
                        parsed = parse_segment_filename(member.name)
                        if not parsed:
                            continue
                        speaker, start_time = parsed
                        seg_meta = segment_map.get((speaker, round(start_time, 2)))
                        if not seg_meta:
                            matches = [
                                s for s in usable_segments
                                if s.get("speaker") == speaker
                                and abs(s.get("start", 0.0) - start_time) < 0.1
                            ]
                            if not matches:
                                continue
                            seg_meta = matches[0]

                        while len(pending) >= max_inflight:
                            drain_completed()

                        segment_bytes = tf.extractfile(member).read()
                        filename_parts = member.name.split("_")
                        try:
                            index = int(filename_parts[-2])
                        except (ValueError, IndexError):
                            index = 0
                        rng_seed = make_segment_seed(self.seed, video_id, speaker, index)

                        future = executor.submit(
                            process_segment,
                            video_id,
                            self.language,
                            segment_bytes,
                            seg_meta,
                            speaker,
                            index,
                            self.chunk_cfg,
                            self.silence_cfg,
                            rng_seed,
                            self.output_format,
                        )
                        future_info[future] = is_val
                        pending.add(future)

                processed_videos += 1
                if self.on_progress:
                    self.on_progress(processed_videos, total_videos)

            while pending:
                drain_completed()

        return self.stats

    def close(self) -> None:
        """Close all writers and manifests."""
        for writer in self.writers.values():
            writer.close()
        self.train_manifest.close()
        self.val_manifest.close()

    def save_stats(self) -> Path:
        """Save processing stats to JSON."""
        stats_dir = self.output_dir / "stats"
        stats_dir.mkdir(parents=True, exist_ok=True)
        stats_path = stats_dir / f"{self.language}_stats_{self.shard_prefix}.json"
        stats_path.write_text(json.dumps(self.stats.to_dict(), indent=2))
        return stats_path


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""R2 upload module with retry logic and progress tracking."""

from __future__ import annotations

import logging
import os
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, Optional

import boto3
from boto3.s3.transfer import TransferConfig
from botocore.exceptions import ClientError

logger = logging.getLogger(__name__)

@dataclass
class UploadResult:
    """Result of an upload operation."""

    success: bool = True
    uploaded_keys: list[str] = field(default_factory=list)
    failed_files: list[str] = field(default_factory=list)
    total_bytes: int = 0
    elapsed_sec: float = 0.0
    errors: list[str] = field(default_factory=list)

class R2Uploader:
    """Handles uploads to R2 with retry logic and progress tracking."""

    def __init__(
        self,
        endpoint_url: str,
        access_key_id: str,
        secret_access_key: str,
        bucket: str,
        max_retries: int = 3,
        retry_delay_sec: float = 5.0,
        max_concurrency: int = 4,
    ):
        self.bucket = bucket
        self.max_retries = max_retries
        self.retry_delay_sec = retry_delay_sec

        self.client = boto3.client(
            "s3",
            endpoint_url=endpoint_url,
            aws_access_key_id=access_key_id,
            aws_secret_access_key=secret_access_key,
        )

        self.transfer_config = TransferConfig(
            multipart_threshold=8 * 1024 * 1024,  # 8MB
            multipart_chunksize=64 * 1024 * 1024,  # 64MB
            max_concurrency=max_concurrency,
            use_threads=True,
        )

        # Progress callback
        self._on_progress: Optional[Callable[[str, int, int], None]] = None

    def set_progress_callback(
        self, callback: Optional[Callable[[str, int, int], None]]
    ) -> None:
        """Set callback for upload progress: callback(filename, bytes_transferred, total_bytes)."""
        self._on_progress = callback

    def upload_file(
        self,
        local_path: Path,
        remote_key: str,
        metadata: Optional[dict[str, str]] = None,
    ) -> bool:
        """Upload a single file with retry logic.

        Args:
            local_path: Path to local file
            remote_key: Remote key in bucket
            metadata: Optional metadata to attach

        Returns:
            True if upload succeeded, False otherwise
        """
        if not local_path.exists():
            logger.error(f"File not found: {local_path}")
            return False

        file_size = local_path.stat().st_size

        for attempt in range(self.max_retries):
            try:
                extra_args = {}
                if metadata:
                    extra_args["Metadata"] = metadata

                # Create progress callback for this file
                callback = None
                if self._on_progress:
                    def make_callback(filename: str, total: int):
                        bytes_transferred = 0

                        def progress(chunk_bytes: int):
                            nonlocal bytes_transferred
                            bytes_transferred += chunk_bytes
                            if self._on_progress:
                                self._on_progress(filename, bytes_transferred, total)

                        return progress

                    callback = make_callback(local_path.name, file_size)

                self.client.upload_file(
                    str(local_path),
                    self.bucket,
                    remote_key,
                    Config=self.transfer_config,
                    ExtraArgs=extra_args if extra_args else None,
                    Callback=callback,
                )

                logger.debug(f"Uploaded {local_path.name} -> {remote_key}")
                return True

            except ClientError as e:
                error_code = e.response.get("Error", {}).get("Code", "Unknown")
                logger.warning(
                    f"Upload attempt {attempt + 1}/{self.max_retries} failed for "
                    f"{local_path.name}: {error_code} - {e}"
                )

                if attempt < self.max_retries - 1:
                    delay = self.retry_delay_sec * (2 ** attempt)  # Exponential backoff
                    logger.info(f"Retrying in {delay:.1f}s...")
                    time.sleep(delay)

            except Exception as e:
                logger.error(f"Unexpected error uploading {local_path.name}: {e}")
                if attempt < self.max_retries - 1:
                    time.sleep(self.retry_delay_sec)

        logger.error(f"Failed to upload {local_path} after {self.max_retries} attempts")
        return False

    def upload_directory(
        self,
        local_dir: Path,
        remote_prefix: str,
        pattern: str = "*",
    ) -> UploadResult:
        """Upload all files from a directory.

        Args:
            local_dir: Local directory to upload from
            remote_prefix: Remote prefix for all files
            pattern: Glob pattern to match files

        Returns:
            UploadResult with details
        """
        result = UploadResult()
        start_time = time.time()

        if not local_dir.exists():
            result.success = False
            result.errors.append(f"Directory not found: {local_dir}")
            return result

        files = list(local_dir.glob(pattern))
        if not files:
            logger.warning(f"No files matching '{pattern}' in {local_dir}")
            return result

        for local_path in sorted(files):
            if not local_path.is_file():
                continue

            remote_key = f"{remote_prefix.rstrip('/')}/{local_path.name}"
            file_size = local_path.stat().st_size

            if self.upload_file(local_path, remote_key):
                result.uploaded_keys.append(remote_key)
                result.total_bytes += file_size
            else:
                result.failed_files.append(str(local_path))

        result.elapsed_sec = time.time() - start_time
        result.success = len(result.failed_files) == 0

        logger.info(
            f"Directory upload complete: {len(result.uploaded_keys)} files, "
            f"{result.total_bytes / (1024**2):.1f}MB in {result.elapsed_sec:.1f}s"
        )

        return result

    def upload_shard(
        self,
        output_dir: Path,
        language: str,
        shard_id: str,
        audio_ext: str = "flac",
    ) -> UploadResult:
        """Upload a complete shard (tar + jsonl for each sample rate).

        This uploads:
        - {language}_16/shard-{shard_id}.tar and .jsonl
        - {language}_24/shard-{shard_id}.tar and .jsonl (if exists)
        - {language}_32/shard-{shard_id}.tar and .jsonl (if exists)
        - {language}_44/shard-{shard_id}.tar and .jsonl (if exists)

        Args:
            output_dir: Base output directory
            language: Language code
            shard_id: Shard identifier
            audio_ext: Audio extension (flac or wav)

        Returns:
            UploadResult with details
        """
        result = UploadResult()
        start_time = time.time()

        sr_map = {
            16000: "16",
            24000: "24",
            32000: "32",
            44100: "44",
        }

        for sr, sr_suffix in sr_map.items():
            local_dir = output_dir / f"{language}_{sr_suffix}"
            if not local_dir.exists():
                continue

            # Upload tar file
            tar_pattern = f"*-{shard_id}*.tar"
            for tar_file in local_dir.glob(tar_pattern):


Summary: auto


sed -n '240,520p' workers/r2_uploader.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
            for tar_file in local_dir.glob(tar_pattern):
                remote_key = f"{language}_{sr_suffix}/{tar_file.name}"
                if self.upload_file(tar_file, remote_key):
                    result.uploaded_keys.append(remote_key)
                    result.total_bytes += tar_file.stat().st_size
                else:
                    result.failed_files.append(str(tar_file))

            # Upload jsonl file
            jsonl_pattern = f"*-{shard_id}*.jsonl"
            for jsonl_file in local_dir.glob(jsonl_pattern):
                remote_key = f"{language}_{sr_suffix}/{jsonl_file.name}"
                if self.upload_file(jsonl_file, remote_key):
                    result.uploaded_keys.append(remote_key)
                    result.total_bytes += jsonl_file.stat().st_size
                else:
                    result.failed_files.append(str(jsonl_file))

        # Upload manifests
        manifest_dir = output_dir / "manifests"
        if manifest_dir.exists():
            for manifest_file in manifest_dir.glob(f"*{shard_id}*"):
                remote_key = f"manifests/{manifest_file.name}"
                if self.upload_file(manifest_file, remote_key):
                    result.uploaded_keys.append(remote_key)
                    result.total_bytes += manifest_file.stat().st_size
                else:
                    result.failed_files.append(str(manifest_file))

        # Upload stats
        stats_dir = output_dir / "stats"
        if stats_dir.exists():
            for stats_file in stats_dir.glob(f"*{shard_id}*"):
                remote_key = f"stats/{stats_file.name}"
                if self.upload_file(stats_file, remote_key):
                    result.uploaded_keys.append(remote_key)
                    result.total_bytes += stats_file.stat().st_size
                else:
                    result.failed_files.append(str(stats_file))

        result.elapsed_sec = time.time() - start_time
        result.success = len(result.failed_files) == 0

        if result.success:
            logger.info(
                f"Shard {shard_id} uploaded: {len(result.uploaded_keys)} files, "
                f"{result.total_bytes / (1024**2):.1f}MB in {result.elapsed_sec:.1f}s"
            )
        else:
            result.errors.append(
                f"Failed to upload {len(result.failed_files)} files for shard {shard_id}"
            )

        return result

    def verify_upload(self, remote_key: str) -> bool:
        """Verify that a file exists in R2.

        Args:
            remote_key: Remote key to verify

        Returns:
            True if file exists
        """
        try:
            self.client.head_object(Bucket=self.bucket, Key=remote_key)
            return True
        except ClientError as e:
            if e.response.get("Error", {}).get("Code") == "404":
                return False
            raise

    def download_file(
        self,
        remote_key: str,
        local_path: Path,
    ) -> bool:
        """Download a file from R2.

        Args:
            remote_key: Remote key to download
            local_path: Local path to save to

        Returns:
            True if download succeeded
        """
        local_path.parent.mkdir(parents=True, exist_ok=True)

        for attempt in range(self.max_retries):
            try:
                self.client.download_file(
                    self.bucket,
                    remote_key,
                    str(local_path),
                    Config=self.transfer_config,
                )
                return True

            except ClientError as e:
                logger.warning(
                    f"Download attempt {attempt + 1}/{self.max_retries} failed: {e}"
                )
                if attempt < self.max_retries - 1:
                    time.sleep(self.retry_delay_sec * (2 ** attempt))

        return False

    def resolve_key(
        self,
        r2_key: str,
        prefix: Optional[str] = None,
    ) -> Optional[str]:
        """Resolve R2 key, trying multiple path variants.

        Args:
            r2_key: R2 key to resolve
            prefix: Optional prefix to try

        Returns:
            Resolved key or None if not found
        """
        if not r2_key:
            return None

        key = r2_key.lstrip("/")
        candidates = [key]

        if prefix:
            cleaned_prefix = prefix.strip("/")
            if not key.startswith(cleaned_prefix):
                candidates.append(f"{cleaned_prefix}/{key}")

        base = os.path.basename(key)
        if base != key:
            candidates.append(base)
            if prefix:
                candidates.append(f"{prefix.strip('/')}/{base}")

        for candidate in candidates:
            try:
                self.client.head_object(Bucket=self.bucket, Key=candidate)
                return candidate
            except ClientError:
                continue

        return None


sed -n '1,260p' workers/storage_manager.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""Storage management for workers - monitors disk usage and performs cleanup."""

from __future__ import annotations

import logging
import os
import shutil
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable, Optional

logger = logging.getLogger(__name__)

@dataclass
class StorageStats:
    """Current storage statistics."""

    total_gb: float = 0.0
    used_gb: float = 0.0
    free_gb: float = 0.0
    usage_percent: float = 0.0
    work_dir_gb: float = 0.0
    output_dir_gb: float = 0.0

@dataclass
class CleanupResult:
    """Result of a cleanup operation."""

    success: bool = True
    freed_gb: float = 0.0
    files_removed: int = 0
    dirs_removed: int = 0
    errors: list[str] = field(default_factory=list)

class StorageManager:
    """Manages disk storage for workers with automatic cleanup."""

    def __init__(
        self,
        work_dir: Path,
        output_dir: Path,
        max_usage_gb: float = 150.0,
        cleanup_threshold_gb: float = 100.0,
        check_interval_sec: int = 60,
    ):
        self.work_dir = work_dir
        self.output_dir = output_dir
        self.max_usage_gb = max_usage_gb
        self.cleanup_threshold_gb = cleanup_threshold_gb
        self.check_interval_sec = check_interval_sec

        self._stop_event = threading.Event()
        self._monitor_thread: Optional[threading.Thread] = None
        self._cleanup_in_progress = False
        self._lock = threading.Lock()

        # Callbacks for health reporting
        self._on_cleanup_start: Optional[Callable[[], None]] = None
        self._on_cleanup_complete: Optional[Callable[[CleanupResult], None]] = None
        self._on_storage_critical: Optional[Callable[[StorageStats], None]] = None

        # Track completed shards for safe cleanup
        self._uploaded_shards: set[str] = set()

    def get_stats(self) -> StorageStats:
        """Get current storage statistics."""
        try:
            # Get filesystem stats for work_dir
            stat = os.statvfs(self.work_dir)
            total_gb = (stat.f_blocks * stat.f_frsize) / (1024 ** 3)
            free_gb = (stat.f_bavail * stat.f_frsize) / (1024 ** 3)
            used_gb = total_gb - free_gb
            usage_percent = (used_gb / total_gb * 100) if total_gb > 0 else 0

            work_dir_gb = self._get_dir_size_gb(self.work_dir)
            output_dir_gb = self._get_dir_size_gb(self.output_dir)

            return StorageStats(
                total_gb=total_gb,
                used_gb=used_gb,
                free_gb=free_gb,
                usage_percent=usage_percent,
                work_dir_gb=work_dir_gb,
                output_dir_gb=output_dir_gb,
            )
        except Exception as e:
            logger.error(f"Failed to get storage stats: {e}")
            return StorageStats()

    def _get_dir_size_gb(self, path: Path) -> float:
        """Get directory size in GB."""
        if not path.exists():
            return 0.0
        total = 0
        try:
            for entry in path.rglob("*"):
                if entry.is_file():
                    try:
                        total += entry.stat().st_size
                    except (OSError, FileNotFoundError):
                        pass
        except Exception as e:
            logger.warning(f"Error calculating dir size for {path}: {e}")
        return total / (1024 ** 3)

    def needs_cleanup(self) -> bool:
        """Check if cleanup is needed based on current usage."""
        stats = self.get_stats()
        return (
            stats.work_dir_gb + stats.output_dir_gb > self.max_usage_gb
            or stats.free_gb < 10.0  # Always clean if less than 10GB free
        )

    def mark_shard_uploaded(self, shard_id: str) -> None:
        """Mark a shard as uploaded (safe to delete local files)."""
        with self._lock:
            self._uploaded_shards.add(shard_id)

    def cleanup(self, force: bool = False) -> CleanupResult:
        """Perform cleanup of old/uploaded data.

        Cleanup priority:
        1. Uploaded shards (output_dir)
        2. Downloaded tars for completed shards (work_dir)
        3. Temporary files
        """
        with self._lock:
            if self._cleanup_in_progress and not force:
                return CleanupResult(success=False, errors=["Cleanup already in progress"])
            self._cleanup_in_progress = True

        if self._on_cleanup_start:
            self._on_cleanup_start()

        result = CleanupResult()
        initial_stats = self.get_stats()
        initial_used = initial_stats.work_dir_gb + initial_stats.output_dir_gb

        try:
            # 1. Clean up uploaded shard data
            for shard_id in list(self._uploaded_shards):
                # Clean shard directory in work_dir
                shard_work_dir = self.work_dir / shard_id
                if shard_work_dir.exists():
                    try:
                        shutil.rmtree(shard_work_dir)
                        result.dirs_removed += 1
                        logger.info(f"Cleaned up work dir for shard {shard_id}")
                    except Exception as e:
                        result.errors.append(f"Failed to clean {shard_work_dir}: {e}")

                # Clean output files for this shard
                for subdir in self.output_dir.iterdir():
                    if subdir.is_dir():
                        for f in subdir.glob(f"*{shard_id}*"):
                            try:
                                f.unlink()
                                result.files_removed += 1
                            except Exception as e:
                                result.errors.append(f"Failed to remove {f}: {e}")

                with self._lock:
                    self._uploaded_shards.discard(shard_id)

            # 2. Clean old temporary files (older than 2 hours)
            cutoff = time.time() - 7200  # 2 hours
            for path in self.work_dir.rglob("*"):
                if path.is_file():
                    try:
                        if path.stat().st_mtime < cutoff:
                            path.unlink()
                            result.files_removed += 1
                    except (OSError, FileNotFoundError):
                        pass

            # 3. Clean empty directories
            for path in sorted(self.work_dir.rglob("*"), reverse=True):
                if path.is_dir():
                    try:
                        if not any(path.iterdir()):
                            path.rmdir()
                            result.dirs_removed += 1
                    except (OSError, FileNotFoundError):
                        pass

            # Calculate freed space
            final_stats = self.get_stats()
            final_used = final_stats.work_dir_gb + final_stats.output_dir_gb
            result.freed_gb = max(0, initial_used - final_used)
            result.success = True

        except Exception as e:
            result.success = False
            result.errors.append(f"Cleanup failed: {e}")
            logger.exception("Cleanup failed")

        finally:
            with self._lock:
                self._cleanup_in_progress = False

        if self._on_cleanup_complete:
            self._on_cleanup_complete(result)

        logger.info(
            f"Cleanup complete: freed {result.freed_gb:.2f}GB, "
            f"removed {result.files_removed} files, {result.dirs_removed} dirs"
        )
        return result

    def cleanup_shard_data(self, shard_id: str, shard_work_dir: Path) -> CleanupResult:
        """Clean up all data for a specific shard after successful upload."""
        result = CleanupResult()

        try:
            # Remove shard work directory (downloads)
            if shard_work_dir.exists():
                shutil.rmtree(shard_work_dir)
                result.dirs_removed += 1
                logger.info(f"Removed shard work dir: {shard_work_dir}")

            # Remove output files for this shard
            for sr_dir in self.output_dir.iterdir():
                if sr_dir.is_dir():
                    for f in sr_dir.glob(f"*-{shard_id}*"):
                        try:
                            f.unlink()
                            result.files_removed += 1
                        except Exception as e:
                            result.errors.append(f"Failed to remove {f}: {e}")

            # Remove manifest entries if they exist
            manifest_dir = self.output_dir / "manifests"
            if manifest_dir.exists():
                for f in manifest_dir.glob(f"*{shard_id}*"):
                    try:
                        f.unlink()
                        result.files_removed += 1
                    except Exception as e:
                        result.errors.append(f"Failed to remove {f}: {e}")

            result.success = True

        except Exception as e:
            result.success = False
            result.errors.append(str(e))
            logger.exception(f"Failed to cleanup shard {shard_id}")

        return result

    def start_monitor(self) -> None:
        """Start background storage monitoring thread."""
        if self._monitor_thread is not None:
            return

        self._stop_event.clear()


sed -n '260,620p' workers/storage_manager.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
        self._stop_event.clear()
        self._monitor_thread = threading.Thread(
            target=self._monitor_loop,
            name="storage-monitor",
            daemon=True,
        )
        self._monitor_thread.start()
        logger.info("Storage monitor started")

    def stop_monitor(self) -> None:
        """Stop background storage monitoring."""
        self._stop_event.set()
        if self._monitor_thread:
            self._monitor_thread.join(timeout=5)
            self._monitor_thread = None
        logger.info("Storage monitor stopped")

    def _monitor_loop(self) -> None:
        """Background loop for storage monitoring."""
        while not self._stop_event.is_set():
            try:
                stats = self.get_stats()
                total_worker_gb = stats.work_dir_gb + stats.output_dir_gb

                # Check if we need cleanup
                if total_worker_gb > self.max_usage_gb or stats.free_gb < 10.0:
                    logger.warning(
                        f"Storage threshold exceeded: "
                        f"worker={total_worker_gb:.1f}GB (max={self.max_usage_gb}GB), "
                        f"free={stats.free_gb:.1f}GB"
                    )

                    if self._on_storage_critical:
                        self._on_storage_critical(stats)

                    # Attempt cleanup
                    cleanup_result = self.cleanup()

                    # Check if cleanup was sufficient
                    if not cleanup_result.success or cleanup_result.freed_gb < 1.0:
                        logger.error(
                            f"Cleanup insufficient: freed only {cleanup_result.freed_gb:.2f}GB"
                        )
                        # Retry cleanup with more aggressive settings
                        time.sleep(5)
                        self.cleanup(force=True)

            except Exception as e:
                logger.exception(f"Storage monitor error: {e}")

            # Wait for next check
            self._stop_event.wait(self.check_interval_sec)

    def set_callbacks(
        self,
        on_cleanup_start: Optional[Callable[[], None]] = None,
        on_cleanup_complete: Optional[Callable[[CleanupResult], None]] = None,
        on_storage_critical: Optional[Callable[[StorageStats], None]] = None,
    ) -> None:
        """Set callbacks for storage events."""
        self._on_cleanup_start = on_cleanup_start
        self._on_cleanup_complete = on_cleanup_complete
        self._on_storage_critical = on_storage_critical


Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/usr/bin/env python3
"""Distributed shard worker for horizontal scaling.

This worker:
1. Claims a batch of videos from Supabase (with row-level locking)
2. Downloads video tars from R2
3. Processes segments to generate shards
4. Uploads shards to R2
5. Updates status in Supabase
6. Cleans up local storage
7. Reports heartbeat and metrics

Key safety features:
- Deterministic shard_id (same videos = same shard = idempotent upload)
- FOR UPDATE SKIP LOCKED prevents duplicate claims
- Lease expiry allows crash recovery
- Storage monitoring prevents disk exhaustion
"""

from __future__ import annotations

import argparse
import hashlib
import json
import logging
import os
import signal
import socket
import sys
import threading
import time
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Optional

import psycopg
from boto3.s3.transfer import TransferConfig

# Setup logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger("worker")

# Import worker modules
from config import WorkerConfig, EnvConfig
from storage_manager import StorageManager, StorageStats
from r2_uploader import R2Uploader
from worker_processing import (
    VideoProcessor,
    ChunkConfig,
    SilenceConfig,
    ProcessingStats,
)

@dataclass(frozen=True)
class TaskRow:
    """A video task from the database."""
    video_id: str
    r2_tar_key: str
    language: str

@dataclass
class WorkerState:
    """Current worker state for heartbeat reporting."""
    status: str = "idle"
    shard_id: Optional[str] = None
    videos_assigned: int = 0
    videos_processed: int = 0
    segments_processed: int = 0
    outputs_generated: int = 0
    last_error: Optional[str] = None
    processing_start: Optional[float] = None

class ShardWorker:
    """Main worker class that orchestrates the processing pipeline."""

    def __init__(
        self,
        config: WorkerConfig,
        env_config: EnvConfig,
    ):
        self.config = config
        self.env_config = env_config

        # Generate worker ID if not provided
        self.worker_id = config.worker_id or f"{socket.gethostname()}-{os.getpid()}"
        logger.info(f"Worker ID: {self.worker_id}")

        # Initialize R2 clients
        self.source_uploader = R2Uploader(
            endpoint_url=env_config.r2_endpoint_url,
            access_key_id=env_config.r2_access_key_id,
            secret_access_key=env_config.r2_secret_access_key,
            bucket=env_config.r2_cleaned_bucket,
            max_retries=config.max_retries,
            retry_delay_sec=config.retry_delay_sec,
            max_concurrency=config.download_workers,
        )

        self.shard_uploader = R2Uploader(
            endpoint_url=env_config.r2_endpoint_url,
            access_key_id=env_config.r2_access_key_id,
            secret_access_key=env_config.r2_secret_access_key,
            bucket=env_config.r2_bucket_shards,
            max_retries=config.max_retries,
            retry_delay_sec=config.retry_delay_sec,
            max_concurrency=config.upload_workers,
        )

        # Initialize storage manager
        self.storage_manager = StorageManager(
            work_dir=config.work_dir,
            output_dir=config.output_dir,
            max_usage_gb=config.max_disk_usage_gb,
            cleanup_threshold_gb=config.cleanup_threshold_gb,
            check_interval_sec=config.cleanup_check_interval_sec,
        )

        # Worker state
        self.state = WorkerState()
        self._state_lock = threading.Lock()
        self._stop_event = threading.Event()
        self._heartbeat_thread: Optional[threading.Thread] = None

        # Setup signal handlers
        signal.signal(signal.SIGTERM, self._handle_signal)
        signal.signal(signal.SIGINT, self._handle_signal)

    def _handle_signal(self, signum, frame):
        """Handle shutdown signals gracefully."""
        logger.info(f"Received signal {signum}, initiating graceful shutdown...")
        self._stop_event.set()

    def _update_state(self, **kwargs) -> None:
        """Thread-safe state update."""
        with self._state_lock:
            for key, value in kwargs.items():
                if hasattr(self.state, key):
                    setattr(self.state, key, value)

    def _get_state(self) -> dict:
        """Get current state as dict."""
        with self._state_lock:
            return {
                "status": self.state.status,
                "shard_id": self.state.shard_id,
                "videos_assigned": self.state.videos_assigned,
                "videos_processed": self.state.videos_processed,
                "segments_processed": self.state.segments_processed,
                "outputs_generated": self.state.outputs_generated,
                "last_error": self.state.last_error,
            }

    def deterministic_shard_id(self, language: str, video_ids: list[str]) -> uuid.UUID:
        """Generate deterministic shard ID from language + sorted video IDs.

        This ensures that re-processing the same videos produces the same shard_id,
        making uploads idempotent (overwrites same R2 key instead of creating duplicates).
        """
        joined = f"{language}:" + ",".join(sorted(video_ids))
        digest = hashlib.sha1(joined.encode("utf-8")).hexdigest()
        return uuid.uuid5(uuid.NAMESPACE_URL, digest)

    def claim_videos(self, conn) -> list[TaskRow]:
        """Claim a batch of pending videos with row-level locking.

        Uses FOR UPDATE SKIP LOCKED to prevent duplicate claims across workers.
        """
        with conn.cursor() as cur:
            cur.execute(
                """
                with picked as (
                    select video_id
                    from video_tasks
                    where language = %s
                      and status = 'pending'
                    order by video_id
                    limit %s
                    for update skip locked
                )
                update video_tasks
                set status = 'claimed',
                    claimed_by = %s,
                    lease_expires_at = now() + (%s || ' minutes')::interval,
                    attempts = attempts + 1,
                    updated_at = now()
                where video_id in (select video_id from picked)
                returning video_id, r2_tar_key, language
                """,
                (
                    self.config.language,
                    self.config.target_videos,
                    self.worker_id,
                    str(self.config.lease_minutes),
                ),
            )
            rows = cur.fetchall()
        return [TaskRow(*row) for row in rows]

    def create_shard_run(
        self,
        conn,
        shard_id: uuid.UUID,
        assigned_count: int,
    ) -> None:
        """Create shard_runs entry for tracking."""
        with conn.cursor() as cur:
            cur.execute(
                """
                insert into shard_runs (
                    shard_id, language, target_video_count, assigned_video_count,
                    status, claimed_by, lease_expires_at, shard_prefix, output_format
                ) values (
                    %s, %s, %s, %s,
                    'claimed', %s, now() + (%s || ' minutes')::interval, %s, %s
                )
                on conflict (shard_id) do update set
                    status = 'claimed',
                    claimed_by = excluded.claimed_by,
                    lease_expires_at = excluded.lease_expires_at,
                    updated_at = now()
                """,
                (
                    str(shard_id),
                    self.config.language,
                    self.config.target_videos,
                    assigned_count,
                    self.worker_id,
                    str(self.config.lease_minutes),
                    f"{self.config.language}/{shard_id}",
                    self.config.output_format,
                ),
            )

    def update_task_status(
        self,
        conn,
        video_ids: list[str],
        status: str,
        shard_id: Optional[uuid.UUID] = None,
        error: Optional[str] = None,
    ) -> None:
        """Update status for video tasks."""
        with conn.cursor() as cur:
            cur.execute(
                """
                update video_tasks
                set status = %s,
                    shard_id = %s,
                    last_error = %s,
                    updated_at = now()
                where video_id = any(%s)
                """,


Output:
Exit code: 0
Wall time: 0 seconds
Output:
                """,
                (status, str(shard_id) if shard_id else None, error, video_ids),
            )

    def update_shard_status(
        self,
        conn,
        shard_id: uuid.UUID,
        status: str,
        metrics: Optional[dict] = None,
    ) -> None:
        """Update shard_runs status."""
        with conn.cursor() as cur:
            cur.execute(
                """
                update shard_runs
                set status = %s,
                    metrics = coalesce(%s, metrics),
                    updated_at = now()
                where shard_id = %s
                """,
                (status, json.dumps(metrics) if metrics else None, str(shard_id)),
            )

    def renew_lease(self, conn, shard_id: uuid.UUID, video_ids: list[str]) -> None:
        """Renew lease for current shard and videos."""
        with conn.cursor() as cur:
            # Renew shard lease
            cur.execute(
                """
                update shard_runs
                set lease_expires_at = now() + (%s || ' minutes')::interval,
                    updated_at = now()
                where shard_id = %s
                """,
                (str(self.config.lease_minutes), str(shard_id)),
            )
            # Renew video leases
            cur.execute(
                """
                update video_tasks
                set lease_expires_at = now() + (%s || ' minutes')::interval,
                    updated_at = now()
                where video_id = any(%s)
                """,
                (str(self.config.lease_minutes), video_ids),
            )

    def send_heartbeat(self, conn) -> None:
        """Send heartbeat to database."""
        state = self._get_state()
        storage = self.storage_manager.get_stats()

        with conn.cursor() as cur:
            cur.execute(
                """
                insert into worker_heartbeats (
                    worker_id, hostname, instance_type, language, status,
                    shard_id, videos_assigned, videos_processed,
                    segments_processed, outputs_generated, last_error, last_heartbeat
                ) values (
                    %s, %s, %s, %s, %s,
                    %s, %s, %s,
                    %s, %s, %s, now()
                )
                on conflict (worker_id) do update set
                    status = excluded.status,
                    shard_id = excluded.shard_id,
                    videos_assigned = excluded.videos_assigned,
                    videos_processed = excluded.videos_processed,
                    segments_processed = excluded.segments_processed,
                    outputs_generated = excluded.outputs_generated,
                    last_error = excluded.last_error,
                    last_heartbeat = now()
                """,
                (
                    self.worker_id,
                    socket.gethostname(),
                    os.environ.get("INSTANCE_TYPE", "unknown"),
                    self.config.language,
                    state["status"],
                    state["shard_id"],
                    state["videos_assigned"],
                    state["videos_processed"],
                    state["segments_processed"],
                    state["outputs_generated"],
                    state["last_error"],
                ),
            )

    def _heartbeat_loop(self) -> None:
        """Background heartbeat loop."""
        while not self._stop_event.is_set():
            try:
                with psycopg.connect(self.env_config.database_url) as conn:
                    self.send_heartbeat(conn)
                    conn.commit()
            except Exception as e:
                logger.warning(f"Heartbeat failed: {e}")

            self._stop_event.wait(self.config.heartbeat_interval_sec)

    def start_heartbeat(self) -> None:
        """Start background heartbeat thread."""
        if self._heartbeat_thread is not None:
            return
        self._heartbeat_thread = threading.Thread(
            target=self._heartbeat_loop,
            name="heartbeat",
            daemon=True,
        )
        self._heartbeat_thread.start()
        logger.info("Heartbeat thread started")

    def stop_heartbeat(self) -> None:
        """Stop heartbeat thread."""
        self._stop_event.set()
        if self._heartbeat_thread:
            self._heartbeat_thread.join(timeout=5)
            self._heartbeat_thread = None

    def download_videos(
        self,
        tasks: list[TaskRow],
        download_dir: Path,
    ) -> dict[str, Path]:
        """Download video tars from R2."""
        download_dir.mkdir(parents=True, exist_ok=True)
        download_paths: dict[str, Path] = {}

        transfer_config = TransferConfig(
            multipart_threshold=8 * 1024 * 1024,
            multipart_chunksize=64 * 1024 * 1024,
            max_concurrency=max(4, self.config.download_workers),
            use_threads=True,
        )

        for i, task in enumerate(tasks):
            if self._stop_event.is_set():
                break

            # Check storage before download
            if self.storage_manager.needs_cleanup():
                logger.warning("Storage threshold exceeded, running cleanup...")
                self.storage_manager.cleanup()

            local_path = download_dir / f"{task.video_id}.tar"
            if local_path.exists():
                download_paths[task.video_id] = local_path
                continue

            # Resolve R2 key
            resolved = self.source_uploader.resolve_key(
                task.r2_tar_key,
                self.env_config.r2_prefix,
            )
            if not resolved:
                logger.warning(f"Could not resolve R2 key: {task.r2_tar_key}")
                continue

            # Download
            try:
                if self.source_uploader.download_file(resolved, local_path):
                    download_paths[task.video_id] = local_path
                    logger.debug(f"Downloaded {task.video_id} ({i+1}/{len(tasks)})")
                else:
                    logger.error(f"Failed to download {task.video_id}")
            except Exception as e:
                logger.error(f"Download error for {task.video_id}: {e}")

            # Update progress
            self._update_state(videos_processed=len(download_paths))

        return download_paths

    def process_shard(
        self,
        shard_id: uuid.UUID,
        download_paths: dict[str, Path],
    ) -> ProcessingStats:
        """Process downloaded videos into shards."""
        self.config.output_dir.mkdir(parents=True, exist_ok=True)

        chunk_cfg = ChunkConfig(
            enabled=self.config.chunk_enabled,
            min_sec=self.config.chunk_min_sec,
            max_sec=self.config.chunk_max_sec,
            threshold_sec=self.config.chunk_threshold_sec,
            all_buckets=self.config.chunk_all_buckets,
        )
        silence_cfg = SilenceConfig(
            enabled=self.config.silence_enabled,
            frame_sec=self.config.silence_frame_ms / 1000.0,
            hop_sec=self.config.silence_hop_ms / 1000.0,
            search_sec=self.config.silence_search_sec,
            percentile=self.config.silence_percentile,
            db_drop=self.config.silence_db_drop,
        )

        # Use shard_id as prefix for deterministic output naming
        shard_prefix = f"shard-{str(shard_id)[:8]}"

        def on_progress(processed: int, total: int):
            self._update_state(videos_processed=processed)
            # Check storage during processing
            if processed % 10 == 0 and self.storage_manager.needs_cleanup():
                logger.warning("Storage threshold during processing, running cleanup...")
                self.storage_manager.cleanup()

        processor = VideoProcessor(
            output_dir=self.config.output_dir,
            language=self.config.language,
            shard_prefix=shard_prefix,
            chunk_cfg=chunk_cfg,
            silence_cfg=silence_cfg,
            output_format=self.config.output_format,
            target_shard_size_mb=self.config.target_shard_size_mb,
            segment_workers=self.config.segment_workers,
            seed=self.config.seed,
            val_ratio=self.config.val_ratio,
            on_progress=on_progress,
        )

        try:
            stats = processor.process_videos(download_paths)
            processor.close()
            processor.save_stats()

            # Update state with processing stats
            total_outputs = sum(stats.segments_kept.values())
            self._update_state(
                segments_processed=stats.segments_total,
                outputs_generated=total_outputs,
            )

            return stats
        except Exception as e:
            processor.close()
            raise

    def upload_shard(
        self,
        shard_id: uuid.UUID,
    ) -> bool:
        """Upload processed shard to R2."""
        shard_prefix = f"shard-{str(shard_id)[:8]}"

        result = self.shard_uploader.upload_shard(
            output_dir=self.config.output_dir,
            language=self.config.language,
            shard_id=str(shard_id)[:8],
            audio_ext="wav" if self.config.output_format == "wav" else "flac",
        )

        if result.success:
            logger.info(
                f"Uploaded shard {shard_id}: {len(result.uploaded_keys)} files, "
                f"{result.total_bytes / (1024**3):.2f}GB"
            )
        else:
            logger.error(f"Upload failed for shard {shard_id}: {result.errors}")

        return result.success

    def process_one_shard(self) -> bool:
        """Process a single shard from claim to upload.

        Returns True if a shard was processed, False if no work available.
        """
        # Claim videos
        self._update_state(status="claiming")
        with psycopg.connect(self.env_config.database_url) as conn:
            conn.execute("begin")
            tasks = self.claim_videos(conn)

            if not tasks:
                conn.execute("commit")
                logger.info("No pending videos available")
                return False

            video_ids = [t.video_id for t in tasks]
            shard_id = self.deterministic_shard_id(self.config.language, video_ids)

            logger.info(f"Claimed {len(tasks)} videos, shard_id={shard_id}")

            # Create shard run and mark videos as processing
            self.create_shard_run(conn, shard_id, len(tasks))
            self.update_task_status(conn, video_ids, "processing", shard_id=shard_id)
            conn.execute("commit")

        self._update_state(
            status="processing",
            shard_id=str(shard_id),
            videos_assigned=len(tasks),
            videos_processed=0,
            processing_start=time.time(),
        )

        try:
            # Download
            self._update_state(status="downloading")
            download_dir = self.config.work_dir / str(shard_id)
            download_paths = self.download_videos(tasks, download_dir)

            if not download_paths:
                raise RuntimeError("No videos downloaded successfully")

            # Renew lease before processing
            with psycopg.connect(self.env_config.database_url) as conn:
                self.renew_lease(conn, shard_id, video_ids)
                conn.commit()

            # Process
            self._update_state(status="processing")
            stats = self.process_shard(shard_id, download_paths)

            # Renew lease before upload
            with psycopg.connect(self.env_config.database_url) as conn:
                self.renew_lease(conn, shard_id, video_ids)
                conn.commit()

            # Upload
            self._update_state(status="uploading")
            if not self.upload_shard(shard_id):
                raise RuntimeError("Upload failed")

            # Mark completed
            with psycopg.connect(self.env_config.database_url) as conn:
                self.update_shard_status(conn, shard_id, "completed", metrics=stats.to_dict())
                self.update_task_status(conn, video_ids, "completed", shard_id=shard_id)
                conn.commit()

            logger.info(f"Completed shard {shard_id} with {len(download_paths)} videos")

            # Cleanup
            self._update_state(status="cleanup")
            self.storage_manager.cleanup_shard_data(str(shard_id), download_dir)
            self.storage_manager.mark_shard_uploaded(str(shard_id))

            self._update_state(status="idle", shard_id=None)
            return True

        except Exception as e:
            logger.exception(f"Failed to process shard {shard_id}")
            self._update_state(status="failed", last_error=str(e))

            # Mark as failed in database
            try:
                with psycopg.connect(self.env_config.database_url) as conn:
                    self.update_shard_status(conn, shard_id, "failed")
                    self.update_task_status(
                        conn, video_ids, "failed", shard_id=shard_id, error=str(e)
                    )
                    conn.commit()
            except Exception as db_err:
                logger.error(f"Failed to update failure status: {db_err}")

            return False

    def run(self) -> None:
        """Main worker loop."""


Output:
Exit code: 0
Wall time: 0 seconds
Output:
        """Main worker loop."""
        logger.info(f"Starting worker for language={self.config.language}")

        # Validate config
        errors = self.env_config.validate()
        if errors:
            for error in errors:
                logger.error(f"Config error: {error}")
            raise SystemExit(1)

        # Start background threads
        self.start_heartbeat()
        self.storage_manager.start_monitor()

        # Set storage callbacks
        def on_storage_critical(stats: StorageStats):
            logger.warning(
                f"Storage critical: {stats.work_dir_gb + stats.output_dir_gb:.1f}GB used, "
                f"{stats.free_gb:.1f}GB free"
            )
            self._update_state(last_error="Storage critical")

        self.storage_manager.set_callbacks(on_storage_critical=on_storage_critical)

        try:
            while not self._stop_event.is_set():
                try:
                    processed = self.process_one_shard()

                    if self.config.once:
                        logger.info("Single shard mode, exiting")
                        break

                    if not processed:
                        # No work available, wait before retry
                        logger.info("No work available, sleeping 30s...")
                        self._stop_event.wait(30)

                except Exception as e:
                    logger.exception(f"Error in main loop: {e}")
                    self._update_state(status="error", last_error=str(e))
                    # Wait before retry
                    self._stop_event.wait(60)

        finally:
            logger.info("Shutting down worker...")
            self.stop_heartbeat()
            self.storage_manager.stop_monitor()

            # Final heartbeat with idle status
            try:
                self._update_state(status="stopped")
                with psycopg.connect(self.env_config.database_url) as conn:
                    self.send_heartbeat(conn)
                    conn.commit()
            except Exception:
                pass

            logger.info("Worker shutdown complete")

def parse_args() -> argparse.Namespace:
    """Parse command line arguments."""
    parser = argparse.ArgumentParser(description="Distributed shard worker")

    # Required
    parser.add_argument("--language", required=True, help="Language to process")

    # Worker identity
    parser.add_argument("--worker-id", default=None, help="Worker ID (auto-generated if not set)")

    # Processing settings
    parser.add_argument("--target-videos", type=int, default=100, help="Videos per shard")
    parser.add_argument("--lease-minutes", type=int, default=60, help="Lease duration")

    # Directories
    parser.add_argument("--work-dir", type=Path, default=Path("/tmp/worker"))
    parser.add_argument("--output-dir", type=Path, default=Path("/tmp/worker_output"))

    # Storage
    parser.add_argument("--max-disk-gb", type=float, default=150.0, help="Max disk usage GB")
    parser.add_argument("--cleanup-threshold-gb", type=float, default=100.0)

    # Parallelism
    parser.add_argument("--download-workers", type=int, default=8)
    parser.add_argument("--segment-workers", type=int, default=8)
    parser.add_argument("--upload-workers", type=int, default=4)

    # Output
    parser.add_argument("--output-format", choices=["flac", "wav"], default="flac")
    parser.add_argument("--shard-size-mb", type=int, default=5120)

    # Run mode
    parser.add_argument("--once", action="store_true", help="Process single shard and exit")
    parser.add_argument("--dry-run", action="store_true", help="Don't actually process")

    # Config file
    parser.add_argument("--env-file", type=Path, default=Path(".env"))

    return parser.parse_args()

def main():
    args = parse_args()

    # Load environment config
    if args.env_file.exists():
        env_config = EnvConfig.from_file(args.env_file)
    else:
        env_config = EnvConfig.from_env_vars()

    # Create worker config from args
    worker_config = WorkerConfig(
        worker_id=args.worker_id or "",
        language=args.language,
        target_videos=args.target_videos,
        lease_minutes=args.lease_minutes,
        work_dir=args.work_dir,
        output_dir=args.output_dir,
        max_disk_usage_gb=args.max_disk_gb,
        cleanup_threshold_gb=args.cleanup_threshold_gb,
        download_workers=args.download_workers,
        segment_workers=args.segment_workers,
        upload_workers=args.upload_workers,
        output_format=args.output_format,
        target_shard_size_mb=args.shard_size_mb,
        once=args.once,
        dry_run=args.dry_run,
    )

    worker = ShardWorker(worker_config, env_config)
    worker.run()

if __name__ == "__main__":
    main()


**Assessing shard ID usage and naming consistency**

Noticed a subtle mismatch: the full UUID shard_id is stored in the database, but only the first 8 characters are used for shard file naming and prefixes. The upload prefix uses language sampling rate without shard_id as a folder, relying on shard-<first8> filenames for uniqueness, which should be consistent but may affect idempotency guarantees.


**Planning to use full UUIDs for shard filenames and adjusting shard splitting**

Decided to switch shard filenames to use the full UUID to avoid rare collisions and ensure idempotency. Also identified the need to disable shard splitting by size to support fixed-count shards, proposing to modify ShardWriter to skip splitting when target size is non-positive or explicitly disabled for consistent single-shard outputs per worker.


sed -n '1,220p' workers/release_stale_claims.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/usr/bin/env python3
"""Release stale claims from crashed workers.

This script should be run periodically (e.g., via cron) to:
1. Release video_tasks with expired leases back to 'pending'
2. Mark associated shard_runs as 'stale'
3. Log recovery actions for debugging

Usage:
    python release_stale_claims.py [--dry-run] [--env-file .env]
"""

from __future__ import annotations

import argparse
import logging
from datetime import datetime
from pathlib import Path

import psycopg

from config import EnvConfig

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger("stale-recovery")

def release_stale_video_tasks(conn, dry_run: bool = False) -> int:
    """Release video_tasks with expired leases back to pending."""
    with conn.cursor() as cur:
        if dry_run:
            cur.execute(
                """
                select video_id, claimed_by, lease_expires_at, status
                from video_tasks
                where status in ('claimed', 'processing')
                  and lease_expires_at < now()
                """
            )
            stale = cur.fetchall()
            for row in stale:
                logger.info(
                    f"[DRY-RUN] Would release video {row[0]} "
                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
                )
            return len(stale)
        else:
            cur.execute(
                """
                update video_tasks
                set status = 'pending',
                    claimed_by = null,
                    lease_expires_at = null,
                    updated_at = now()
                where status in ('claimed', 'processing')
                  and lease_expires_at < now()
                returning video_id, claimed_by
                """
            )
            released = cur.fetchall()
            for video_id, claimed_by in released:
                logger.info(f"Released video {video_id} (was claimed by {claimed_by})")
            return len(released)

def mark_stale_shards(conn, dry_run: bool = False) -> int:
    """Mark shard_runs with expired leases as stale."""
    with conn.cursor() as cur:
        if dry_run:
            cur.execute(
                """
                select shard_id, claimed_by, lease_expires_at, status
                from shard_runs
                where status in ('claimed', 'processing', 'uploading')
                  and lease_expires_at < now()
                """
            )
            stale = cur.fetchall()
            for row in stale:
                logger.info(
                    f"[DRY-RUN] Would mark shard {row[0]} as stale "
                    f"(claimed_by={row[1]}, expired={row[2]}, status={row[3]})"
                )
            return len(stale)
        else:
            cur.execute(
                """
                update shard_runs
                set status = 'stale',
                    updated_at = now()
                where status in ('claimed', 'processing', 'uploading')
                  and lease_expires_at < now()
                returning shard_id, claimed_by
                """
            )
            stale = cur.fetchall()
            for shard_id, claimed_by in stale:
                logger.info(f"Marked shard {shard_id} as stale (was claimed by {claimed_by})")
            return len(stale)

def log_recovery_event(conn, video_ids: int, shard_ids: int) -> None:
    """Log recovery event for audit trail."""
    with conn.cursor() as cur:
        cur.execute(
            """
            insert into video_task_events (event, details)
            values ('stale_recovery', %s)
            """,
            (f'{{"videos_released": {video_ids}, "shards_marked_stale": {shard_ids}}}',),
        )

def get_stats(conn) -> dict:
    """Get current task statistics."""
    with conn.cursor() as cur:
        cur.execute(
            """
            select status, count(*)
            from video_tasks
            group by status
            """
        )
        video_stats = dict(cur.fetchall())

        cur.execute(
            """
            select status, count(*)
            from shard_runs
            group by status
            """
        )
        shard_stats = dict(cur.fetchall())

        cur.execute(
            """
            select count(distinct worker_id)
            from worker_heartbeats
            where last_heartbeat > now() - interval '5 minutes'
            """
        )
        active_workers = cur.fetchone()[0]

    return {
        "video_tasks": video_stats,
        "shard_runs": shard_stats,
        "active_workers": active_workers,
    }

def main():
    parser = argparse.ArgumentParser(description="Release stale claims from crashed workers")
    parser.add_argument("--dry-run", action="store_true", help="Show what would be done")
    parser.add_argument("--env-file", type=Path, default=Path(".env"))
    parser.add_argument("--stats-only", action="store_true", help="Only show stats, don't recover")
    args = parser.parse_args()

    # Load config
    if args.env_file.exists():
        env_config = EnvConfig.from_file(args.env_file)
    else:
        env_config = EnvConfig.from_env_vars()

    if not env_config.database_url:
        logger.error("DATABASE_URL not configured")
        raise SystemExit(1)

    with psycopg.connect(env_config.database_url) as conn:
        # Show current stats
        stats = get_stats(conn)
        logger.info("=== Current Statistics ===")
        logger.info(f"Video tasks: {stats['video_tasks']}")
        logger.info(f"Shard runs: {stats['shard_runs']}")
        logger.info(f"Active workers (last 5 min): {stats['active_workers']}")

        if args.stats_only:
            return

        logger.info("")
        logger.info("=== Releasing Stale Claims ===")

        # Release stale claims
        videos_released = release_stale_video_tasks(conn, args.dry_run)
        shards_marked = mark_stale_shards(conn, args.dry_run)

        if not args.dry_run:
            if videos_released > 0 or shards_marked > 0:
                log_recovery_event(conn, videos_released, shards_marked)
            conn.commit()

        logger.info("")
        logger.info(f"Released {videos_released} video tasks")
        logger.info(f"Marked {shards_marked} shards as stale")

        # Show updated stats
        if not args.dry_run and (videos_released > 0 or shards_marked > 0):
            stats = get_stats(conn)
            logger.info("")
            logger.info("=== Updated Statistics ===")
            logger.info(f"Video tasks: {stats['video_tasks']}")
            logger.info(f"Shard runs: {stats['shard_runs']}")

if __name__ == "__main__":
    main()


Output:
Exit code: 0
Wall time: 0 seconds
Output:
# Distributed Shard Workers

Horizontally scalable audio processing pipeline for converting video segments into training-ready shards.

## Overview

This system processes videos language-by-language using multiple workers that:
1. **Claim** batches of videos from Supabase (with row-level locking - no duplicates)
2. **Download** video tars from R2 (source bucket)
3. **Process** audio segments (resample, chunk, compute metrics)
4. **Upload** shards to R2 (destination bucket)
5. **Report** heartbeats and metrics to Supabase

### Key Safety Features

- **FOR UPDATE SKIP LOCKED**: Prevents duplicate claims across workers
- **Deterministic shard_id**: Same videos = same shard = idempotent uploads
- **Lease expiry**: Crashed workers' claims are automatically released
- **Storage monitoring**: Prevents disk exhaustion during processing

## Quick Start

### 1. Setup Database

Run the schema in Supabase:

```bash
# Apply schema
psql $DATABASE_URL < schema.sql
```

### 2. Seed Video Tasks

```bash
# Seed video_tasks table for a language
python seed_video_tasks.py --language telugu

# With limit for testing
python seed_video_tasks.py --language telugu --limit 1000 --dry-run
```

### 3. Configure Environment

Create `.env` file:

```bash
# Database
DATABASE_URL=postgresql://user:pass@host:5432/db

# R2 Storage
R2_ENDPOINT_URL=https://xxx.r2.cloudflarestorage.com
R2_ACCESS_KEY_ID=your_key
R2_SECRET_ACCESS_KEY=your_secret
R2_CLEANED_BUCKET=cleaned-videos    # Source bucket with video tars
R2_BUCKET_SHARDS=training-shards    # Destination bucket for shards
R2_PREFIX=                          # Optional prefix for source keys
```

### 4. Run Worker

```bash
# Single worker
python worker.py --language telugu

# Process single shard and exit (for testing)
python worker.py --language telugu --once

# With custom settings
python worker.py \
    --language telugu \
    --target-videos 100 \
    --max-disk-gb 150 \
    --download-workers 8 \
    --segment-workers 8
```

### 5. Docker Deployment

```bash
# Build image from project root
docker build -f workers/Dockerfile.cpu -t shard-worker .

# Run single worker
docker run -d \
    --name worker-1 \
    -e LANGUAGE=telugu \
    -e DATABASE_URL="..." \
    -e R2_ENDPOINT_URL="..." \
    -e R2_ACCESS_KEY_ID="..." \
    -e R2_SECRET_ACCESS_KEY="..." \
    -e R2_CLEANED_BUCKET="..." \
    -e R2_BUCKET_SHARDS="..." \
    shard-worker

# Or use docker-compose
cd workers
docker-compose up -d --scale worker=4
```

## Architecture

### Files

```
workers/
├── worker.py              # Main worker orchestration
├── worker_processing.py   # Audio processing (adapted from prepare_shards.py)
├── storage_manager.py     # Disk monitoring and cleanup
├── r2_uploader.py         # R2 upload with retry logic
├── config.py              # Configuration management
├── seed_video_tasks.py    # Seed video_tasks table
├── release_stale_claims.py # Recover from crashed workers
├── schema.sql             # Supabase schema
├── Dockerfile.cpu         # Docker image
├── docker-compose.yml     # Local testing
├── entrypoint.sh          # Docker entrypoint
└── requirements.txt       # Python dependencies
```

### Database Schema

#### `video_tasks`
Tracks each video exactly once.

| Column | Type | Description |
|--------|------|-------------|
| video_id | text PK | YouTube video ID |
| language | text | Language code |
| r2_tar_key | text | R2 key for source tar |
| status | text | pending/claimed/processing/completed/failed |
| shard_id | uuid | Assigned shard |
| claimed_by | text | Worker ID |
| lease_expires_at | timestamptz | Claim expiry |
| attempts | int | Attempt count |

#### `shard_runs`
One row per shard assignment.

| Column | Type | Description |
|--------|------|-------------|
| shard_id | uuid PK | Deterministic from videos |
| language | text | Language code |
| target_video_count | int | Requested videos |
| assigned_video_count | int | Actual videos |
| status | text | claimed/processing/uploading/completed/failed/stale |
| metrics | jsonb | Processing stats |

#### `worker_heartbeats`
Worker status and metrics.

| Column | Type | Description |
|--------|------|-------------|
| worker_id | text PK | Worker identifier |
| status | text | idle/claiming/processing/uploading/failed |
| videos_processed | int | Current progress |
| segments_processed | int | Total segments |
| outputs_generated | int | Output count |
| last_heartbeat | timestamptz | Last update |

### Processing Flow

```
┌─────────────────────────────────────────────────────────────────┐
│                         Worker Loop                              │
├─────────────────────────────────────────────────────────────────┤
│  1. Claim Videos (FOR UPDATE SKIP LOCKED)                       │
│     └─> video_tasks.status = 'claimed'                          │
│                                                                  │
│  2. Create shard_runs entry                                      │
│     └─> shard_id = hash(language + sorted(video_ids))           │
│                                                                  │
│  3. Download tars from R2                                        │
│     └─> Check storage, cleanup if needed                        │
│                                                                  │
│  4. Process segments                                             │
│     ├─> Decode FLAC                                             │
│     ├─> Compute metrics (LUFS, spectral)                        │
│     ├─> Chunk long segments (silence-aware)                     │
│     ├─> Resample to target rates (16k, 24k, 32k, 44.1k)        │
│     └─> Write to tar + jsonl shards                             │
│                                                                  │
│  5. Upload shards to R2                                          │
│     └─> Retry with exponential backoff                          │
│                                                                  │
│  6. Finalize                                                     │
│     ├─> video_tasks.status = 'completed'                        │
│     ├─> shard_runs.status = 'completed'                         │
│     └─> Cleanup local files                                      │
│                                                                  │
│  7. Repeat                                                       │
└─────────────────────────────────────────────────────────────────┘
```

## Scaling

### Horizontal Scaling (Recommended)

Start multiple workers, each will claim different videos:

```bash


**Reviewing config usage in worker and entrypoint**

Noted that the worker code uses the config but skips the dry_run setting; I'll verify if dry_run appears in the docs and check entrypoint and docker-compose to confirm correct environment setup next.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/bin/bash
set -e

# Entrypoint script for shard worker container
# Handles environment setup, validation, and graceful shutdown

echo "=== Shard Worker Entrypoint ==="
echo "Hostname: $(hostname)"
echo "Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"

# Validate required environment variables
validate_env() {
    local missing=()

    [[ -z "$LANGUAGE" ]] && missing+=("LANGUAGE")
    [[ -z "$DATABASE_URL" ]] && missing+=("DATABASE_URL")
    [[ -z "$R2_ENDPOINT_URL" ]] && missing+=("R2_ENDPOINT_URL")
    [[ -z "$R2_ACCESS_KEY_ID" ]] && missing+=("R2_ACCESS_KEY_ID")
    [[ -z "$R2_SECRET_ACCESS_KEY" ]] && missing+=("R2_SECRET_ACCESS_KEY")
    [[ -z "$R2_CLEANED_BUCKET" ]] && missing+=("R2_CLEANED_BUCKET")
    [[ -z "$R2_BUCKET_SHARDS" ]] && missing+=("R2_BUCKET_SHARDS")

    if [[ ${#missing[@]} -gt 0 ]]; then
        echo "ERROR: Missing required environment variables:"
        for var in "${missing[@]}"; do
            echo "  - $var"
        done
        exit 1
    fi

    echo "Environment validation passed"
}

# Create work directories with proper permissions
setup_directories() {
    local work_dir="${WORK_DIR:-/tmp/worker}"
    local output_dir="${OUTPUT_DIR:-/tmp/worker_output}"

    echo "Creating directories..."
    mkdir -p "$work_dir" "$output_dir"

    # Check available disk space
    local available_gb=$(df -BG "$work_dir" | awk 'NR==2 {print $4}' | sed 's/G//')
    echo "Available disk space: ${available_gb}GB"

    if [[ "$available_gb" -lt 50 ]]; then
        echo "WARNING: Less than 50GB available, may run into storage issues"
    fi
}

# Handle graceful shutdown
shutdown_handler() {
    echo "Received shutdown signal, forwarding to worker..."
    if [[ -n "$WORKER_PID" ]]; then
        kill -TERM "$WORKER_PID" 2>/dev/null || true
        wait "$WORKER_PID" 2>/dev/null || true
    fi
    echo "Shutdown complete"
    exit 0
}

trap shutdown_handler SIGTERM SIGINT

# Main
validate_env
setup_directories

# Build command line arguments
CMD_ARGS=(
    "--language" "$LANGUAGE"
    "--work-dir" "${WORK_DIR:-/tmp/worker}"
    "--output-dir" "${OUTPUT_DIR:-/tmp/worker_output}"
)

[[ -n "$WORKER_ID" ]] && CMD_ARGS+=("--worker-id" "$WORKER_ID")
[[ -n "$TARGET_VIDEOS" ]] && CMD_ARGS+=("--target-videos" "$TARGET_VIDEOS")
[[ -n "$LEASE_MINUTES" ]] && CMD_ARGS+=("--lease-minutes" "$LEASE_MINUTES")
[[ -n "$MAX_DISK_GB" ]] && CMD_ARGS+=("--max-disk-gb" "$MAX_DISK_GB")
[[ -n "$CLEANUP_THRESHOLD_GB" ]] && CMD_ARGS+=("--cleanup-threshold-gb" "$CLEANUP_THRESHOLD_GB")
[[ -n "$DOWNLOAD_WORKERS" ]] && CMD_ARGS+=("--download-workers" "$DOWNLOAD_WORKERS")
[[ -n "$SEGMENT_WORKERS" ]] && CMD_ARGS+=("--segment-workers" "$SEGMENT_WORKERS")
[[ -n "$UPLOAD_WORKERS" ]] && CMD_ARGS+=("--upload-workers" "$UPLOAD_WORKERS")
[[ -n "$OUTPUT_FORMAT" ]] && CMD_ARGS+=("--output-format" "$OUTPUT_FORMAT")
[[ -n "$SHARD_SIZE_MB" ]] && CMD_ARGS+=("--shard-size-mb" "$SHARD_SIZE_MB")
[[ "$ONCE" == "true" || "$ONCE" == "1" ]] && CMD_ARGS+=("--once")

echo "Starting worker with args: ${CMD_ARGS[*]}"
echo "==================================="

# Run worker in background so we can handle signals
cd /app
python -m workers.worker "${CMD_ARGS[@]}" &
WORKER_PID=$!

# Wait for worker to complete
wait "$WORKER_PID"
exit_code=$?

echo "Worker exited with code: $exit_code"
exit $exit_code


sed -n '1,200p' workers/docker-compose.yml


Output:
Exit code: 0
Wall time: 0 seconds
Output:
# Docker Compose for local testing of shard workers
# Usage: docker-compose up -d --scale worker=4

version: '3.8'

services:
  worker:
    build:
      context: ..
      dockerfile: workers/Dockerfile.cpu
    environment:
      # Required
      - LANGUAGE=${LANGUAGE:-telugu}
      - DATABASE_URL=${DATABASE_URL}
      - R2_ENDPOINT_URL=${R2_ENDPOINT_URL}
      - R2_ACCESS_KEY_ID=${R2_ACCESS_KEY_ID}
      - R2_SECRET_ACCESS_KEY=${R2_SECRET_ACCESS_KEY}
      - R2_CLEANED_BUCKET=${R2_CLEANED_BUCKET}
      - R2_BUCKET_SHARDS=${R2_BUCKET_SHARDS}
      - R2_PREFIX=${R2_PREFIX:-}

      # Optional tuning
      - TARGET_VIDEOS=${TARGET_VIDEOS:-100}
      - LEASE_MINUTES=${LEASE_MINUTES:-60}
      - MAX_DISK_GB=${MAX_DISK_GB:-150}
      - CLEANUP_THRESHOLD_GB=${CLEANUP_THRESHOLD_GB:-100}
      - DOWNLOAD_WORKERS=${DOWNLOAD_WORKERS:-8}
      - SEGMENT_WORKERS=${SEGMENT_WORKERS:-8}
      - UPLOAD_WORKERS=${UPLOAD_WORKERS:-4}
      - OUTPUT_FORMAT=${OUTPUT_FORMAT:-flac}
      - SHARD_SIZE_MB=${SHARD_SIZE_MB:-5120}

      # Instance info
      - INSTANCE_TYPE=${INSTANCE_TYPE:-local}

    volumes:
      # Mount work directories for persistence/inspection
      - worker_data:/tmp/worker
      - worker_output:/tmp/worker_output

    deploy:
      resources:
        limits:
          memory: 16G
        reservations:
          memory: 8G

    restart: unless-stopped

    # Optional: limit to specific CPUs
    # cpuset: "0-7"

volumes:
  worker_data:
  worker_output:

# Example usage:
# 1. Copy .env.example to .env and fill in credentials
# 2. docker-compose build
# 3. docker-compose up -d --scale worker=4  # Start 4 workers
# 4. docker-compose logs -f worker  # Watch logs
# 5. docker-compose down  # Stop all workers


**Identifying bug in download handling**

The worker incorrectly marks all videos as completed even if some downloads fail; I'll fix this by tracking successful downloads and marking failed ones appropriately instead of marking all as done.


**Refactoring shard handling and lease renewal**

I'm redesigning the workflow to compute shard_id only after downloads succeed, updating task statuses to reflect successes and failures separately to avoid duplicates and maintain idempotency. I'll also implement periodic lease renewal during long processing to prevent stale claims and fix shard naming inconsistencies by using full UUIDs to ensure proper cleanup and no filename collisions.


**Fixing imports to use relative paths**

I'll update all imports within the workers package to use relative imports, ensuring module resolution works correctly when running as a module and avoiding import errors.


sed -n '1,120p' workers/seed_video_tasks.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/usr/bin/env python3
"""Seed video_tasks from the existing videos table (Supabase/Postgres)."""

from __future__ import annotations

import argparse
from pathlib import Path

import psycopg
import sys

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))

from prepare_shards import load_env

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Seed video_tasks table.")
    parser.add_argument("--language", required=True)
    parser.add_argument("--limit", type=int, default=None)
    parser.add_argument("--dry-run", action="store_true")
    return parser.parse_args()

def main() -> None:
    args = parse_args()
    env = load_env(Path(".env"))
    db_url = env.get("DATABASE_URL")
    if not db_url:
        raise SystemExit("DATABASE_URL missing in .env")

    with psycopg.connect(db_url) as conn:
        with conn.cursor() as cur:
            query = """
                select youtube_id, r2_tar_key, language
                from videos
                where lower(language) = %s
                  and r2_tar_key is not null
                  and status = 'COMPLETED'
            """
            params = [args.language.lower()]
            if args.limit:
                query += " limit %s"
                params.append(args.limit)
            cur.execute(query, params)
            rows = cur.fetchall()

    if args.dry_run:
        print(f"Would insert {len(rows)} rows for language={args.language}")
        return

    with psycopg.connect(db_url) as conn:
        with conn.cursor() as cur:
            for video_id, r2_tar_key, language in rows:
                cur.execute(
                    """
                    insert into video_tasks (video_id, language, r2_tar_key)
                    values (%s, %s, %s)
                    on conflict (video_id) do nothing
                    """,
                    (video_id, language, r2_tar_key),
                )
        conn.commit()

    print(f"Inserted {len(rows)} rows for language={args.language}")

if __name__ == "__main__":
    main()


**Noting missing dependencies and import adjustments**

Identified missing packages (`scipy`, `pyloudnorm`) in requirements despite usage in code; planning to update `workers/requirements.txt` accordingly and switch to relative imports in worker modules to align with the existing package structure and entrypoint usage.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
FROM python:3.10-slim

# Install system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
    libsndfile1 \
    ffmpeg \
    curl \
    && rm -rf /var/lib/apt/lists/*

# Set working directory
WORKDIR /app

# Copy requirements first for better caching
COPY workers/requirements.txt /app/workers/

# Install Python dependencies
RUN pip install --no-cache-dir -r workers/requirements.txt

# Copy worker code
COPY workers/ /app/workers/

# Make entrypoint executable
RUN chmod +x /app/workers/entrypoint.sh

# Create work directories
RUN mkdir -p /tmp/worker /tmp/worker_output

# Set environment defaults
ENV PYTHONUNBUFFERED=1 \
    PYTHONDONTWRITEBYTECODE=1 \
    WORK_DIR=/tmp/worker \
    OUTPUT_DIR=/tmp/worker_output

# Health check - verify worker can import successfully
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
    CMD python -c "import sys; sys.path.insert(0, '/app'); from workers.config import WorkerConfig; print('OK')" || exit 1

# Use entrypoint script
ENTRYPOINT ["/app/workers/entrypoint.sh"]


**Clarifying worker imports and dependencies**

The worker_processing module is standalone and doesn't require prepare_shards.py, but it uses packages missing from requirements that need adding. The worker imports config from the workers package, so absolute imports like `from workers.config` suit running as a module (`python -m workers.worker`), matching the entrypoint. The README uses script execution (`python worker.py`), which would fail with absolute imports, so it needs updating to run as a module or adjust sys.path for compatibility.


**Detailing import fixes, dependency updates, and shard ID changes**

I'm refining worker import statements to use robust absolute imports with sys.path adjustment for script execution, updating README and entrypoint accordingly. I'll add missing dependencies (`scipy`, `pyloudnorm`) to requirements and switch shard ID usage to full UUIDs for naming and cleanup consistency. Also planning config changes to disable shard size splitting by default per requirements and preparing end-to-end validation steps including schema setup, seeding, and shard upload verification.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
# Core dependencies
boto3>=1.28.0
psycopg[binary]>=3.1.0
numpy>=1.24.0
soundfile>=0.12.0
scipy>=1.11.0
tqdm>=4.65.0

# Audio metrics (optional but recommended)
pyloudnorm>=0.1.0


**Updating imports and shard splitting logic**

I'm switching imports to absolute paths by ensuring the root is on sys.path, updating worker.py and release_stale_claims.py accordingly, and adjusting README commands to use `python -m workers.worker`. Also modifying shard splitting in worker_processing to skip splitting when target shard size is zero or negative, updating config defaults and usage to reflect this.


**Finalizing shard ID handling and lease renewal**

I'm updating shard ID usage to use full UUID strings consistently across processing, uploading, and cleanup, ensuring name patterns align. Also adding lease renewal threads during shard processing to keep leases refreshed, adjusting task status updates to recover missing downloads by resetting to pending, and enabling zero shard size to disable splitting.


Input:
*** Begin Patch
*** Update File: workers/worker.py
@@
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any, Optional
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Optional
@@
-# Import worker modules
-from config import WorkerConfig, EnvConfig
-from storage_manager import StorageManager, StorageStats
-from r2_uploader import R2Uploader
-from worker_processing import (
-    VideoProcessor,
-    ChunkConfig,
-    SilenceConfig,
-    ProcessingStats,
-)
+if __package__ in (None, ""):
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+# Import worker modules
+from workers.config import WorkerConfig, EnvConfig
+from workers.storage_manager import StorageManager, StorageStats
+from workers.r2_uploader import R2Uploader
+from workers.worker_processing import (
+    VideoProcessor,
+    ChunkConfig,
+    SilenceConfig,
+    ProcessingStats,
+)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM workers/worker.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}