     1→"""Worker processing module - adapted from prepare_shards.py for worker use.
     2→
     3→This module handles the core audio processing logic:
     4→- Extract segments from video tars
     5→- Compute audio metrics (LUFS, spectral, etc.)
     6→- Chunk long segments
     7→- Resample to target rates
     8→- Write to shards
     9→
    10→The processing logic is identical to prepare_shards.py to ensure consistent output.
    11→"""
    12→
    13→from __future__ import annotations
    14→
    15→import io
    16→import json
    17→import math
    18→import os
    19→import random
    20→import tarfile
    21→import threading
    22→import time
    23→import zlib
    24→from collections import defaultdict
    25→from dataclasses import dataclass
    26→from pathlib import Path
    27→from typing import Any, Callable, Optional
    28→from concurrent.futures import ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED
    29→
    30→import numpy as np
    31→import soundfile as sf
    32→from scipy import signal
    33→
    34→try:
    35→    import pyloudnorm as pyln
    36→except ImportError:
    37→    pyln = None
    38→
    39→
    40→# Caches for performance
    41→_window_cache: dict[int, np.ndarray] = {}
    42→_freq_cache: dict[tuple[int, int], np.ndarray] = {}
    43→_cache_lock = threading.Lock()
    44→_lufs_local = threading.local()
    45→
    46→
    47→@dataclass(frozen=True)
    48→class ChunkConfig:
    49→    """Configuration for chunking long segments."""
    50→    enabled: bool
    51→    min_sec: float
    52→    max_sec: float
    53→    threshold_sec: float
    54→    all_buckets: bool
    55→
    56→
    57→@dataclass(frozen=True)
    58→class SilenceConfig:
    59→    """Configuration for silence-aware chunk boundaries."""
    60→    enabled: bool
    61→    frame_sec: float
    62→    hop_sec: float
    63→    search_sec: float
    64→    percentile: float
    65→    db_drop: float
    66→
    67→
    68→@dataclass(frozen=True)
    69→class SegmentOutput:
    70→    """Output segment with audio bytes and metadata."""
    71→    sample_rate: int
    72→    audio_bytes: bytes
    73→    metadata: dict[str, Any]
    74→
    75→
    76→@dataclass
    77→class ProcessingStats:
    78→    """Statistics from processing."""
    79→    videos: int = 0
    80→    segments_total: int = 0
    81→    segments_kept: dict = None
    82→    dropped_too_short: int = 0
    83→    dropped_too_long: int = 0
    84→    dropped_clipped: int = 0
    85→    segments_chunked: int = 0
    86→    chunks_emitted: int = 0
    87→    native_sr_buckets: dict = None
    88→
    89→    def __post_init__(self):
    90→        if self.segments_kept is None:
    91→            self.segments_kept = {16000: 0, 24000: 0, 32000: 0, 44100: 0}
    92→        if self.native_sr_buckets is None:
    93→            self.native_sr_buckets = {}
    94→
    95→    def to_dict(self) -> dict:
    96→        return {
    97→            "videos": self.videos,
    98→            "segments_total": self.segments_total,
    99→            "segments_kept": self.segments_kept,
   100→            "dropped_too_short": self.dropped_too_short,
   101→            "dropped_too_long": self.dropped_too_long,
   102→            "dropped_clipped": self.dropped_clipped,
   103→            "segments_chunked": self.segments_chunked,
   104→            "chunks_emitted": self.chunks_emitted,
   105→            "native_sr_buckets": self.native_sr_buckets,
   106→        }
   107→
   108→
   109→def get_hann_window(length: int) -> np.ndarray:
   110→    window = _window_cache.get(length)
   111→    if window is not None:
   112→        return window
   113→    with _cache_lock:
   114→        window = _window_cache.get(length)
   115→        if window is None:
   116→            window = np.hanning(length)
   117→            _window_cache[length] = window
   118→    return window
   119→
   120→
   121→def get_rfftfreq(length: int, sr: int) -> np.ndarray:
   122→    key = (length, sr)
   123→    freqs = _freq_cache.get(key)
   124→    if freqs is not None:
   125→        return freqs
   126→    with _cache_lock:
   127→        freqs = _freq_cache.get(key)
   128→        if freqs is None:
   129→            freqs = np.fft.rfftfreq(length, d=1.0 / sr)
   130→            _freq_cache[key] = freqs
   131→    return freqs
   132→
   133→
   134→def get_lufs_meter(sr: int) -> Optional[Any]:
   135→    if pyln is None:
   136→        return None
   137→    meters = getattr(_lufs_local, "meters", None)
   138→    if meters is None:
   139→        meters = {}
   140→        _lufs_local.meters = meters
   141→    meter = meters.get(sr)
   142→    if meter is None:
   143→        meter = pyln.Meter(sr)
   144→        meters[sr] = meter
   145→    return meter
   146→
   147→
   148→def normalize(value: float, min_val: float, max_val: float) -> float:
   149→    if max_val <= min_val:
   150→        return 0.0
   151→    return max(0.0, min(1.0, (value - min_val) / (max_val - min_val)))
   152→
   153→
   154→def compute_spectral_metrics(audio: np.ndarray, sr: int) -> tuple[float, float, float]:
   155→    """Compute spectral metrics: rolloff, effective nyquist, high freq ratio."""
   156→    if audio.size == 0:
   157→        return 0.0, 0.0, 0.0
   158→
   159→    max_samples = min(audio.size, sr * 5)
   160→    window = get_hann_window(max_samples)
   161→    chunk = audio[:max_samples] * window
   162→    spec = np.fft.rfft(chunk)
   163→    power = np.abs(spec) ** 2
   164→    freqs = get_rfftfreq(max_samples, sr)
   165→    total_energy = float(np.sum(power)) + 1e-12
   166→    cumulative = np.cumsum(power)
   167→    rolloff_idx = int(np.searchsorted(cumulative, 0.85 * cumulative[-1]))
   168→    rolloff_hz = float(freqs[min(rolloff_idx, len(freqs) - 1)])
   169→
   170→    power_db = 10.0 * np.log10(power + 1e-12)
   171→    threshold = power_db.max() - 60.0
   172→    above = np.where(power_db >= threshold)[0]
   173→    effective_nyquist_hz = float(freqs[above[-1]]) if above.size > 0 else 0.0
   174→
   175→    high_freq_energy = float(np.sum(power[freqs >= 8000.0]))
   176→    high_freq_ratio = high_freq_energy / total_energy
   177→
   178→    return rolloff_hz, effective_nyquist_hz, high_freq_ratio
   179→
   180→
   181→def classify_native_bucket(effective_nyquist_hz: float) -> str:
   182→    """Classify audio bandwidth based on effective Nyquist frequency."""
   183→    if effective_nyquist_hz < 8000:
   184→        return "<16k"
   185→    if effective_nyquist_hz < 12000:
   186→        return "16-24k"
   187→    if effective_nyquist_hz < 16000:
   188→        return "24-32k"
   189→    if effective_nyquist_hz < 22050:
   190→        return "32-44k"
   191→    return ">=44k"
   192→
   193→
   194→def compute_audio_metrics(audio: np.ndarray, sr: int) -> dict[str, float]:
   195→    """Compute all audio metrics for a segment."""
   196→    audio = audio.astype(np.float32, copy=False)
   197→    rms = float(np.sqrt(np.mean(audio ** 2) + 1e-12))
   198→    loudness_rms_db = 20.0 * math.log10(rms + 1e-12)
   199→    clipping_fraction = float(np.mean(np.abs(audio) > 0.99))
   200→    dc_offset = float(np.mean(audio))
   201→    noise_floor = float(np.percentile(np.abs(audio), 10))
   202→    snr_proxy = 20.0 * math.log10((rms + 1e-9) / (noise_floor + 1e-9))
   203→
   204→    loudness_lufs = -100.0
   205→    if pyln is not None:
   206→        try:
   207→            meter = get_lufs_meter(sr)
   208→            if meter is not None:
   209→                loudness_lufs = float(meter.integrated_loudness(audio))
   210→        except Exception:
   211→            loudness_lufs = -100.0
   212→
   213→    rolloff_hz, effective_nyquist_hz, high_freq_ratio = compute_spectral_metrics(audio, sr)
   214→
   215→    return {
   216→        "loudness_lufs": loudness_lufs,
   217→        "loudness_rms_db": loudness_rms_db,
   218→        "clipping_fraction": clipping_fraction,
   219→        "dc_offset": dc_offset,
   220→        "snr_proxy": snr_proxy,
   221→        "spectral_rolloff_hz": rolloff_hz,
   222→        "effective_nyquist_hz": effective_nyquist_hz,
   223→        "high_freq_energy_ratio": high_freq_ratio,
   224→    }
   225→
   226→
   227→def compute_quality_score(metrics: dict[str, float], has_music: bool) -> float:
   228→    """Compute overall quality score."""
   229→    score = (
   230→        0.3 * (1.0 - metrics["clipping_fraction"])
   231→        + 0.2 * normalize(metrics["snr_proxy"], 10, 40)
   232→        + 0.2 * (1.0 if not has_music else 0.5)
   233→        + 0.2 * normalize(metrics["loudness_lufs"], -40, -14)
   234→        + 0.1 * (1.0 if abs(metrics["dc_offset"]) < 0.01 else 0.5)
   235→    )
   236→    return max(0.0, min(1.0, score))
   237→
   238→
   239→def resample_audio(audio: np.ndarray, src_sr: int, dst_sr: int) -> np.ndarray:
   240→    """Resample audio to target sample rate."""
   241→    if src_sr == dst_sr:
   242→        return audio
   243→    gcd = math.gcd(src_sr, dst_sr)
   244→    up = dst_sr // gcd
   245→    down = src_sr // gcd
   246→    return signal.resample_poly(audio, up, down).astype(np.float32, copy=False)
   247→
   248→
   249→def encode_audio(audio: np.ndarray, sr: int, output_format: str) -> bytes:
   250→    """Encode audio to bytes."""
   251→    buf = io.BytesIO()
   252→    if output_format == "wav":
   253→        sf.write(buf, audio, sr, format="WAV", subtype="PCM_16")
   254→    else:
   255→        sf.write(buf, audio, sr, format="FLAC", subtype="PCM_16")
   256→    return buf.getvalue()
   257→
   258→
   259→def duration_bucket(duration_sec: float) -> Optional[tuple[str, float]]:
   260→    """Get duration bucket and weight."""
   261→    if duration_sec < 3.0:
   262→        return None
   263→    if duration_sec < 6.0:
   264→        return "3-6s", 0.3
   265→    if duration_sec < 12.0:
   266→        return "6-12s", 1.0
   267→    if duration_sec < 24.0:
   268→        return "12-24s", 0.8
   269→    if duration_sec <= 60.0:
   270→        return "24-60s", 0.5
   271→    return None
   272→
   273→
   274→def compute_frame_rms(
   275→    audio: np.ndarray, sr: int, frame_sec: float, hop_sec: float
   276→) -> tuple[np.ndarray, np.ndarray]:
   277→    """Compute frame-level RMS for silence detection."""
   278→    frame_len = max(1, int(round(frame_sec * sr)))
   279→    hop_len = max(1, int(round(hop_sec * sr)))
   280→    if audio.size < frame_len:
   281→        return np.array([], dtype=np.float32), np.array([], dtype=np.float32)
   282→
   283→    if not audio.flags["C_CONTIGUOUS"]:
   284→        audio = np.ascontiguousarray(audio)
   285→
   286→    num_frames = 1 + (audio.size - frame_len) // hop_len
   287→    if num_frames <= 0:
   288→        return np.array([], dtype=np.float32), np.array([], dtype=np.float32)
   289→
   290→    stride = audio.strides[0]
   291→    frames = np.lib.stride_tricks.as_strided(
   292→        audio,
   293→        shape=(num_frames, frame_len),
   294→        strides=(hop_len * stride, stride),
   295→    )
   296→    rms = np.sqrt(np.mean(frames * frames, axis=1) + 1e-12)
   297→    times = (np.arange(num_frames, dtype=np.float32) * hop_len + (frame_len / 2)) / sr
   298→    return times.astype(np.float32), rms.astype(np.float32)
   299→
   300→
   301→def find_silence_boundary(
   302→    times: np.ndarray,
   303→    rms: np.ndarray,
   304→    search_start: float,
   305→    search_end: float,
   306→    threshold_rms: float,
   307→    median_db: float,
   308→    db_drop: float,
   309→) -> Optional[float]:
   310→    """Find optimal silence boundary for chunking."""
   311→    if times.size == 0:
   312→        return None
   313→    mask = (times >= search_start) & (times <= search_end)
   314→    if not np.any(mask):
   315→        return None
   316→    subset = rms[mask]
   317→    min_idx = int(np.argmin(subset))
   318→    min_rms = float(subset[min_idx])
   319→    min_time = float(times[mask][min_idx])
   320→    min_db = 20.0 * math.log10(min_rms + 1e-12)
   321→    if min_rms <= threshold_rms or (median_db - min_db) >= db_drop:
   322→        return min_time
   323→    return None
   324→
   325→
   326→def make_chunk_windows(
   327→    audio: np.ndarray,
   328→    sr: int,
   329→    cfg: ChunkConfig,
   330→    silence_cfg: SilenceConfig,
   331→    rng: random.Random,
   332→) -> list[tuple[float, float]]:
   333→    """Generate chunk windows for a segment."""
   334→    duration_sec = float(audio.shape[0] / sr)
   335→    if not cfg.enabled or duration_sec < cfg.threshold_sec:
   336→        return [(0.0, duration_sec)]
   337→
   338→    if not silence_cfg.enabled:
   339→        windows: list[tuple[float, float]] = []
   340→        offset = 0.0
   341→        while duration_sec - offset >= cfg.min_sec:
   342→            remaining = duration_sec - offset
   343→            if remaining <= cfg.max_sec:
   344→                length = remaining
   345→            else:
   346→                length = rng.uniform(cfg.min_sec, cfg.max_sec)
   347→                if remaining - length < cfg.min_sec:
   348→                    length = remaining
   349→            windows.append((offset, length))
   350→            offset += length
   351→        return windows
   352→
   353→    times, rms = compute_frame_rms(
   354→        audio, sr, silence_cfg.frame_sec, silence_cfg.hop_sec
   355→    )
   356→    if times.size == 0:
   357→        return [(0.0, duration_sec)]
   358→
   359→    threshold_rms = float(np.quantile(rms, silence_cfg.percentile / 100.0))
   360→    median_db = 20.0 * math.log10(float(np.median(rms)) + 1e-12)
   361→
   362→    windows: list[tuple[float, float]] = []
   363→    offset = 0.0
   364→    while duration_sec - offset > cfg.max_sec:
   365→        remaining = duration_sec - offset
   366→        latest_cut = duration_sec - cfg.min_sec
   367→        if offset + cfg.min_sec >= latest_cut:
   368→            break
   369→        target_len = rng.uniform(cfg.min_sec, cfg.max_sec)
   370→        if remaining - target_len < cfg.min_sec:
   371→            target_len = remaining - cfg.min_sec
   372→        target_end = offset + target_len
   373→        search_low = max(offset + cfg.min_sec, target_end - silence_cfg.search_sec)
   374→        search_high = min(offset + cfg.max_sec, target_end + silence_cfg.search_sec, latest_cut)
   375→        if search_high <= search_low:
   376→            search_low = offset + cfg.min_sec
   377→            search_high = min(offset + cfg.max_sec, latest_cut)
   378→        boundary = find_silence_boundary(
   379→            times,
   380→            rms,
   381→            search_low,
   382→            search_high,
   383→            threshold_rms,
   384→            median_db,
   385→            silence_cfg.db_drop,
   386→        )
   387→        if boundary is None:
   388→            if not windows:
   389→                return [(0.0, duration_sec)]
   390→            windows.append((offset, duration_sec - offset))
   391→            return windows
   392→        windows.append((offset, boundary - offset))
   393→        offset = boundary
   394→
   395→    windows.append((offset, duration_sec - offset))
   396→    return windows
   397→
   398→
   399→def make_segment_seed(base_seed: int, video_id: str, speaker: str, index: int) -> int:
   400→    """Generate deterministic seed for a segment."""
   401→    key = f"{video_id}-{speaker}-{index}-{base_seed}"
   402→    return zlib.adler32(key.encode("utf-8")) & 0xFFFFFFFF
   403→
   404→
   405→def parse_segment_filename(filename: str) -> Optional[tuple[str, float]]:
   406→    """Parse segment filename to extract speaker and start time."""
   407→    base = filename.split("/")[-1]
   408→    if not base.endswith(".flac"):
   409→        return None
   410→
   411→    base = base[:-5]
   412→    parts = base.split("_")
   413→    if len(parts) < 3:
   414→        return None
   415→
   416→    speaker = "_".join(parts[:-2])
   417→    time_range = parts[-1]
   418→    if "-" not in time_range:
   419→        return None
   420→
   421→    try:
   422→        start_str = time_range.split("-")[0]
   423→        start_time = float(start_str)
   424→    except (ValueError, IndexError):
   425→        return None
   426→
   427→    return speaker, start_time
   428→
   429→
   430→def process_segment(
   431→    video_id: str,
   432→    language: str,
   433→    segment_bytes: bytes,
   434→    segment_meta: dict[str, Any],
   435→    speaker: str,
   436→    index: int,
   437→    chunk_cfg: ChunkConfig,
   438→    silence_cfg: SilenceConfig,
   439→    rng_seed: int,
   440→    output_format: str,
   441→) -> tuple[list[SegmentOutput], dict[str, int]]:
   442→    """Process a single segment and return outputs."""
   443→    stats = {
   444→        "dropped_too_short": 0,
   445→        "dropped_too_long": 0,
   446→        "dropped_clipped": 0,
   447→        "chunks_emitted": 0,
   448→        "segments_chunked": 0,
   449→    }
   450→    outputs: list[SegmentOutput] = []
   451→
   452→    try:
   453→        audio, sr = sf.read(io.BytesIO(segment_bytes), dtype="float32")
   454→    except Exception:
   455→        return outputs, stats
   456→
   457→    if audio.ndim > 1:
   458→        audio = np.mean(audio, axis=1)
   459→
   460→    rng = random.Random(rng_seed)
   461→    windows = make_chunk_windows(audio, sr, chunk_cfg, silence_cfg, rng)
   462→
   463→    music_stats = segment_meta.get("music_stats") or {}
   464→    has_music = bool(music_stats.get("has_music", False))
   465→
   466→    def emit_outputs(
   467→        audio_slice: np.ndarray,
   468→        metrics: dict[str, float],
   469→        sample_id: str,
   470→        segment_index: int,
   471→        chunk_index: Optional[int],
   472→        start: Optional[float],
   473→        end: Optional[float],
   474→        original_start: Optional[float],
   475→        original_end: Optional[float],
   476→        target_srs: list[int],
   477→    ) -> None:
   478→        slice_duration = float(audio_slice.shape[0] / sr)
   479→        bucket = duration_bucket(slice_duration)
   480→        if bucket is None:
   481→            if slice_duration < 3.0:
   482→                stats["dropped_too_short"] += 1
   483→            else:
   484→                stats["dropped_too_long"] += 1
   485→            return
   486→        if metrics["clipping_fraction"] > 0.1:
   487→            stats["dropped_clipped"] += 1
   488→            return
   489→
   490→        duration_bucket_name, base_weight = bucket
   491→        sampling_weight = base_weight * (0.5 if has_music else 1.0)
   492→        quality_score = compute_quality_score(metrics, has_music)
   493→        effective_nyquist = metrics["effective_nyquist_hz"]
   494→
   495→        base_metadata = {
   496→            "id": sample_id,
   497→            "video_id": video_id,
   498→            "language": language,
   499→            "segment_index": segment_index,
   500→            "chunk_index": chunk_index,
   501→            "start": start,
   502→            "end": end,
   503→            "original_start": original_start,
   504→            "original_end": original_end,
   505→            "duration_sec": slice_duration,
   506→            "native_sr": sr,
   507→            "effective_nyquist_hz": effective_nyquist,
   508→            "spectral_rolloff_hz": metrics["spectral_rolloff_hz"],
   509→            "high_freq_energy_ratio": metrics["high_freq_energy_ratio"],
   510→            "is_truly_wideband": bool(effective_nyquist >= 12000),
   511→            "native_sr_bucket": classify_native_bucket(effective_nyquist),
   512→            "loudness_lufs": metrics["loudness_lufs"],
   513→            "loudness_rms_db": metrics["loudness_rms_db"],
   514→            "clipping_fraction": metrics["clipping_fraction"],
   515→            "snr_proxy": metrics["snr_proxy"],
   516→            "quality_score": quality_score,
   517→            "music_prob": float(music_stats.get("music_mean", 0.0)),
   518→            "music_stats": music_stats,
   519→            "speaker": speaker,
   520→            "vad_speech_regions": [[0.0, slice_duration]],
   521→            "duration_bucket": duration_bucket_name,
   522→            "sampling_weight": sampling_weight,
   523→        }
   524→
   525→        for target_sr in target_srs:
   526→            resampled = resample_audio(audio_slice, sr, target_sr)
   527→            resampled = np.clip(resampled, -1.0, 1.0)
   528→            audio_bytes = encode_audio(resampled, target_sr, output_format)
   529→            meta = dict(base_metadata)
   530→            meta["sample_rate"] = target_sr
   531→            meta["resample_target_sr"] = target_sr
   532→            outputs.append(SegmentOutput(target_sr, audio_bytes, meta))
   533→
   534→        if target_srs == [16000]:
   535→            stats["chunks_emitted"] += 1
   536→
   537→    sample_id_base = f"{video_id}__{speaker}_{index:04d}"
   538→    use_chunk_suffix = len(windows) > 1
   539→    if use_chunk_suffix:
   540→        stats["segments_chunked"] += 1
   541→
   542→    segment_start = segment_meta.get("start")
   543→    segment_end = segment_meta.get("end")
   544→    original_start = segment_meta.get("original_start")
   545→    original_end = segment_meta.get("original_end")
   546→
   547→    # Emit 16k outputs for all chunks
   548→    for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
   549→        start_idx = int(round(offset_sec * sr))
   550→        end_idx = int(round((offset_sec + chunk_len_sec) * sr))
   551→        audio_slice = audio[start_idx:end_idx]
   552→        if audio_slice.size == 0:
   553→            continue
   554→        slice_duration = float(audio_slice.shape[0] / sr)
   555→        slice_metrics = compute_audio_metrics(audio_slice, sr)
   556→
   557→        chunk_start = float(segment_start) + offset_sec if segment_start is not None else None
   558→        chunk_end = chunk_start + slice_duration if chunk_start is not None else None
   559→        chunk_original_start = (
   560→            float(original_start) + offset_sec if original_start is not None else None
   561→        )
   562→        chunk_original_end = (
   563→            chunk_original_start + slice_duration
   564→            if chunk_original_start is not None
   565→            else None
   566→        )
   567→
   568→        sample_id = sample_id_base
   569→        chunk_index = None
   570→        if use_chunk_suffix:
   571→            sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
   572→            chunk_index = chunk_idx
   573→
   574→        emit_outputs(
   575→            audio_slice,
   576→            slice_metrics,
   577→            sample_id,
   578→            index,
   579→            chunk_index,
   580→            chunk_start,
   581→            chunk_end,
   582→            chunk_original_start,
   583→            chunk_original_end,
   584→            [16000],
   585→        )
   586→
   587→    # High-rate buckets for full segment
   588→    full_metrics = compute_audio_metrics(audio, sr)
   589→    effective_nyquist = full_metrics["effective_nyquist_hz"]
   590→    targets_high: list[int] = []
   591→    if effective_nyquist >= 8000:
   592→        targets_high.append(24000)
   593→    if effective_nyquist >= 12000:
   594→        targets_high.append(32000)
   595→    if effective_nyquist >= 16000:
   596→        targets_high.append(44100)
   597→
   598→    if targets_high:
   599→        if chunk_cfg.all_buckets and use_chunk_suffix:
   600→            for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
   601→                start_idx = int(round(offset_sec * sr))
   602→                end_idx = int(round((offset_sec + chunk_len_sec) * sr))
   603→                audio_slice = audio[start_idx:end_idx]
   604→                if audio_slice.size == 0:
   605→                    continue
   606→                slice_duration = float(audio_slice.shape[0] / sr)
   607→                slice_metrics = compute_audio_metrics(audio_slice, sr)
   608→                slice_effective_nyquist = slice_metrics["effective_nyquist_hz"]
   609→                slice_targets: list[int] = []
   610→                if slice_effective_nyquist >= 8000:
   611→                    slice_targets.append(24000)
   612→                if slice_effective_nyquist >= 12000:
   613→                    slice_targets.append(32000)
   614→                if slice_effective_nyquist >= 16000:
   615→                    slice_targets.append(44100)
   616→                if not slice_targets:
   617→                    continue
   618→
   619→                chunk_start = (
   620→                    float(segment_start) + offset_sec if segment_start is not None else None
   621→                )
   622→                chunk_end = (
   623→                    chunk_start + slice_duration if chunk_start is not None else None
   624→                )
   625→                chunk_original_start = (
   626→                    float(original_start) + offset_sec if original_start is not None else None
   627→                )
   628→                chunk_original_end = (
   629→                    chunk_original_start + slice_duration
   630→                    if chunk_original_start is not None
   631→                    else None
   632→                )
   633→
   634→                sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
   635→                emit_outputs(
   636→                    audio_slice,
   637→                    slice_metrics,
   638→                    sample_id,
   639→                    index,
   640→                    chunk_idx,
   641→                    chunk_start,
   642→                    chunk_end,
   643→                    chunk_original_start,
   644→                    chunk_original_end,
   645→                    slice_targets,
   646→                )
   647→        else:
   648→            emit_outputs(
   649→                audio,
   650→                full_metrics,
   651→                sample_id_base,
   652→                index,
   653→                None,
   654→                segment_start,
   655→                segment_end,
   656→                original_start,
   657→                original_end,
   658→                targets_high,
   659→            )
   660→
   661→    return outputs, stats
   662→
   663→
   664→class ShardWriter:
   665→    """Write samples to tar + jsonl shards."""
   666→
   667→    def __init__(
   668→        self, output_dir: Path, prefix: str, target_size_bytes: int, audio_ext: str
   669→    ) -> None:
   670→        self.output_dir = output_dir
   671→        self.prefix = prefix
   672→        self.target_size_bytes = target_size_bytes
   673→        self.audio_ext = audio_ext
   674→        self.shard_index = 0
   675→        self.current_tar: Optional[tarfile.TarFile] = None
   676→        self.current_jsonl = None
   677→        self.current_size = 0
   678→        self.current_count = 0
   679→        self.current_name = ""
   680→        self.output_dir.mkdir(parents=True, exist_ok=True)
   681→
   682→    def _open_new_shard(self) -> None:
   683→        if self.current_tar:
   684→            self.close()
   685→        shard_name = f"{self.prefix}-{self.shard_index:06d}"
   686→        tar_path = self.output_dir / f"{shard_name}.tar"
   687→        jsonl_path = self.output_dir / f"{shard_name}.jsonl"
   688→        self.current_tar = tarfile.open(tar_path, "w")
   689→        self.current_jsonl = jsonl_path.open("w", encoding="utf-8")
   690→        self.current_size = 0
   691→        self.current_count = 0
   692→        self.current_name = shard_name
   693→        self.shard_index += 1
   694→
   695→    def add_sample(self, sample_id: str, audio_bytes: bytes, metadata: dict[str, Any]) -> str:
   696→        if self.current_tar is None:
   697→            self._open_new_shard()
   698→        if (
   699→            self.target_size_bytes > 0
   700→            and self.current_count > 0
   701→            and self.current_size + len(audio_bytes) > self.target_size_bytes
   702→        ):
   703→            self._open_new_shard()
   704→
   705→        assert self.current_tar is not None
   706→        assert self.current_jsonl is not None
   707→
   708→        arcname = f"{sample_id}.{self.audio_ext}"
   709→        info = tarfile.TarInfo(name=arcname)
   710→        info.size = len(audio_bytes)
   711→        info.mtime = int(time.time())
   712→        self.current_tar.addfile(info, io.BytesIO(audio_bytes))
   713→        self.current_jsonl.write(json.dumps(metadata) + "\n")
   714→
   715→        self.current_size += len(audio_bytes)
   716→        self.current_count += 1
   717→        return self.current_name
   718→
   719→    def close(self) -> None:
   720→        if self.current_tar:
   721→            self.current_tar.close()
   722→        if self.current_jsonl:
   723→            self.current_jsonl.close()
   724→        self.current_tar = None
   725→        self.current_jsonl = None
   726→
   727→
   728→class VideoProcessor:
   729→    """Process videos and write to shards."""
   730→
   731→    def __init__(
   732→        self,
   733→        output_dir: Path,
   734→        language: str,
   735→        shard_prefix: str,
   736→        chunk_cfg: ChunkConfig,
   737→        silence_cfg: SilenceConfig,
   738→        output_format: str = "flac",
   739→        target_shard_size_mb: int = 5120,
   740→        segment_workers: int = 8,
   741→        seed: int = 17,
   742→        val_ratio: float = 0.01,
   743→        on_progress: Optional[Callable[[int, int], None]] = None,
   744→    ):
   745→        self.output_dir = output_dir
   746→        self.language = language
   747→        self.shard_prefix = shard_prefix
   748→        self.chunk_cfg = chunk_cfg
   749→        self.silence_cfg = silence_cfg
   750→        self.output_format = output_format
   751→        self.seed = seed
   752→        self.val_ratio = val_ratio
   753→        self.segment_workers = segment_workers
   754→        self.on_progress = on_progress
   755→
   756→        audio_ext = "wav" if output_format == "wav" else "flac"
   757→        target_bytes = max(0, target_shard_size_mb) * 1024 * 1024
   758→
   759→        self.writers = {
   760→            16000: ShardWriter(
   761→                output_dir / f"{language}_16", shard_prefix, target_bytes, audio_ext
   762→            ),
   763→            24000: ShardWriter(
   764→                output_dir / f"{language}_24", shard_prefix, target_bytes, audio_ext
   765→            ),
   766→            32000: ShardWriter(
   767→                output_dir / f"{language}_32", shard_prefix, target_bytes, audio_ext
   768→            ),
   769→            44100: ShardWriter(
   770→                output_dir / f"{language}_44", shard_prefix, target_bytes, audio_ext
   771→            ),
   772→        }
   773→
   774→        # Manifests
   775→        manifest_dir = output_dir / "manifests"
   776→        manifest_dir.mkdir(parents=True, exist_ok=True)
   777→        self.train_manifest = (manifest_dir / f"{language}_train_{shard_prefix}.jsonl").open(
   778→            "w", encoding="utf-8"
   779→        )
   780→        self.val_manifest = (manifest_dir / f"{language}_val_{shard_prefix}.jsonl").open(
   781→            "w", encoding="utf-8"
   782→        )
   783→
   784→        self.stats = ProcessingStats()
   785→        self.audio_ext = audio_ext
   786→
   787→    def process_videos(
   788→        self,
   789→        download_paths: dict[str, Path],
   790→        val_ids: Optional[set[str]] = None,
   791→    ) -> ProcessingStats:
   792→        """Process all videos and write to shards."""
   793→        if val_ids is None:
   794→            # Determine val set
   795→            video_ids = list(download_paths.keys())
   796→            random.seed(self.seed)
   797→            val_count = max(1, int(len(video_ids) * self.val_ratio)) if len(video_ids) > 1 else 0
   798→            val_ids = set(random.sample(video_ids, val_count)) if val_count > 0 else set()
   799→
   800→        max_inflight = max(128, self.segment_workers * 2)
   801→        pending: set[Any] = set()
   802→        future_info: dict[Any, bool] = {}
   803→
   804→        def handle_future(future) -> None:
   805→            is_val = future_info.pop(future, False)
   806→            outputs, seg_stats = future.result()
   807→            self.stats.dropped_too_short += seg_stats["dropped_too_short"]
   808→            self.stats.dropped_too_long += seg_stats["dropped_too_long"]
   809→            self.stats.dropped_clipped += seg_stats["dropped_clipped"]
   810→            self.stats.segments_chunked += seg_stats.get("segments_chunked", 0)
   811→            self.stats.chunks_emitted += seg_stats.get("chunks_emitted", 0)
   812→
   813→            for out in outputs:
   814→                self.stats.segments_kept[out.sample_rate] += 1
   815→                if out.sample_rate == 16000:
   816→                    bucket = out.metadata.get("native_sr_bucket")
   817→                    if bucket:
   818→                        self.stats.native_sr_buckets[bucket] = (
   819→                            self.stats.native_sr_buckets.get(bucket, 0) + 1
   820→                        )
   821→                writer = self.writers[out.sample_rate]
   822→                shard_name = writer.add_sample(
   823→                    out.metadata["id"], out.audio_bytes, out.metadata
   824→                )
   825→                manifest_entry = dict(out.metadata)
   826→                manifest_entry["shard"] = f"{shard_name}.tar"
   827→                manifest_entry["audio"] = f"{out.metadata['id']}.{self.audio_ext}"
   828→                line = json.dumps(manifest_entry) + "\n"
   829→                if is_val:
   830→                    self.val_manifest.write(line)
   831→                else:
   832→                    self.train_manifest.write(line)
   833→
   834→        def drain_completed() -> None:
   835→            nonlocal pending
   836→            if not pending:
   837→                return
   838→            done, pending = wait(pending, return_when=FIRST_COMPLETED)
   839→            for future in done:
   840→                handle_future(future)
   841→
   842→        total_videos = len(download_paths)
   843→        processed_videos = 0
   844→
   845→        with ThreadPoolExecutor(max_workers=self.segment_workers) as executor:
   846→            for video_id, tar_path in download_paths.items():
   847→                self.stats.videos += 1
   848→                is_val = video_id in val_ids
   849→
   850→                try:
   851→                    tf = tarfile.open(tar_path, "r")
   852→                except Exception as e:
   853→                    continue
   854→
   855→                with tf:
   856→                    try:
   857→                        meta = json.loads(
   858→                            tf.extractfile(f"{video_id}/metadata.json").read()
   859→                        )
   860→                    except Exception:
   861→                        continue
   862→
   863→                    usable_segments = [
   864→                        s for s in meta.get("segments", []) if s.get("status") == "usable"
   865→                    ]
   866→                    segment_map = {
   867→                        (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
   868→                        for seg in usable_segments
   869→                    }
   870→
   871→                    members = [m for m in tf.getmembers() if m.name.endswith(".flac")]
   872→                    self.stats.segments_total += len(members)
   873→
   874→                    for member in members:
   875→                        parsed = parse_segment_filename(member.name)
   876→                        if not parsed:
   877→                            continue
   878→                        speaker, start_time = parsed
   879→                        seg_meta = segment_map.get((speaker, round(start_time, 2)))
   880→                        if not seg_meta:
   881→                            matches = [
   882→                                s for s in usable_segments
   883→                                if s.get("speaker") == speaker
   884→                                and abs(s.get("start", 0.0) - start_time) < 0.1
   885→                            ]
   886→                            if not matches:
   887→                                continue
   888→                            seg_meta = matches[0]
   889→
   890→                        while len(pending) >= max_inflight:
   891→                            drain_completed()
   892→
   893→                        segment_bytes = tf.extractfile(member).read()
   894→                        filename_parts = member.name.split("_")
   895→                        try:
   896→                            index = int(filename_parts[-2])
   897→                        except (ValueError, IndexError):
   898→                            index = 0
   899→                        rng_seed = make_segment_seed(self.seed, video_id, speaker, index)
   900→
   901→                        future = executor.submit(
   902→                            process_segment,
   903→                            video_id,
   904→                            self.language,
   905→                            segment_bytes,
   906→                            seg_meta,
   907→                            speaker,
   908→                            index,
   909→                            self.chunk_cfg,
   910→                            self.silence_cfg,
   911→                            rng_seed,
   912→                            self.output_format,
   913→                        )
   914→                        future_info[future] = is_val
   915→                        pending.add(future)
   916→
   917→                processed_videos += 1
   918→                if self.on_progress:
   919→                    self.on_progress(processed_videos, total_videos)
   920→
   921→            while pending:
   922→                drain_completed()
   923→
   924→        return self.stats
   925→
   926→    def close(self) -> None:
   927→        """Close all writers and manifests."""
   928→        for writer in self.writers.values():
   929→            writer.close()
   930→        self.train_manifest.close()
   931→        self.val_manifest.close()
   932→
   933→    def save_stats(self) -> Path:
   934→        """Save processing stats to JSON."""
   935→        stats_dir = self.output_dir / "stats"
   936→        stats_dir.mkdir(parents=True, exist_ok=True)
   937→        stats_path = stats_dir / f"{self.language}_stats_{self.shard_prefix}.json"
   938→        stats_path.write_text(json.dumps(self.stats.to_dict(), indent=2))
   939→        return stats_path
   940→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

"""Worker processing module - adapted from prepare_shards.py for worker use.

This module handles the core audio processing logic:
- Extract segments from video tars
- Compute audio metrics (LUFS, spectral, etc.)
- Chunk long segments
- Resample to target rates
- Write to shards

The processing logic is identical to prepare_shards.py to ensure consistent output.
"""

from __future__ import annotations

import io
import json
import math
import os
import random
import tarfile
import threading
import time
import zlib
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Optional
from concurrent.futures import ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED

import numpy as np
import soundfile as sf
from scipy import signal

try:
    import pyloudnorm as pyln
except ImportError:
    pyln = None

# Caches for performance
_window_cache: dict[int, np.ndarray] = {}
_freq_cache: dict[tuple[int, int], np.ndarray] = {}
_cache_lock = threading.Lock()
_lufs_local = threading.local()

@dataclass(frozen=True)
class ChunkConfig:
    """Configuration for chunking long segments."""
    enabled: bool
    min_sec: float
    max_sec: float
    threshold_sec: float
    all_buckets: bool

@dataclass(frozen=True)
class SilenceConfig:
    """Configuration for silence-aware chunk boundaries."""
    enabled: bool
    frame_sec: float
    hop_sec: float
    search_sec: float
    percentile: float
    db_drop: float

@dataclass(frozen=True)
class SegmentOutput:
    """Output segment with audio bytes and metadata."""
    sample_rate: int
    audio_bytes: bytes
    metadata: dict[str, Any]

@dataclass
class ProcessingStats:
    """Statistics from processing."""
    videos: int = 0
    segments_total: int = 0
    segments_kept: dict = None
    dropped_too_short: int = 0
    dropped_too_long: int = 0
    dropped_clipped: int = 0
    segments_chunked: int = 0
    chunks_emitted: int = 0
    native_sr_buckets: dict = None

    def __post_init__(self):
        if self.segments_kept is None:
            self.segments_kept = {16000: 0, 24000: 0, 32000: 0, 44100: 0}
        if self.native_sr_buckets is None:
            self.native_sr_buckets = {}

    def to_dict(self) -> dict:
        return {
            "videos": self.videos,
            "segments_total": self.segments_total,
            "segments_kept": self.segments_kept,
            "dropped_too_short": self.dropped_too_short,
            "dropped_too_long": self.dropped_too_long,
            "dropped_clipped": self.dropped_clipped,
            "segments_chunked": self.segments_chunked,
            "chunks_emitted": self.chunks_emitted,
            "native_sr_buckets": self.native_sr_buckets,
        }

def get_hann_window(length: int) -> np.ndarray:
    window = _window_cache.get(length)
    if window is not None:
        return window
    with _cache_lock:
        window = _window_cache.get(length)
        if window is None:
            window = np.hanning(length)
            _window_cache[length] = window
    return window

def get_rfftfreq(length: int, sr: int) -> np.ndarray:
    key = (length, sr)
    freqs = _freq_cache.get(key)
    if freqs is not None:
        return freqs
    with _cache_lock:
        freqs = _freq_cache.get(key)
        if freqs is None:
            freqs = np.fft.rfftfreq(length, d=1.0 / sr)
            _freq_cache[key] = freqs
    return freqs

def get_lufs_meter(sr: int) -> Optional[Any]:
    if pyln is None:
        return None
    meters = getattr(_lufs_local, "meters", None)
    if meters is None:
        meters = {}
        _lufs_local.meters = meters
    meter = meters.get(sr)
    if meter is None:
        meter = pyln.Meter(sr)
        meters[sr] = meter
    return meter

def normalize(value: float, min_val: float, max_val: float) -> float:
    if max_val <= min_val:
        return 0.0
    return max(0.0, min(1.0, (value - min_val) / (max_val - min_val)))

def compute_spectral_metrics(audio: np.ndarray, sr: int) -> tuple[float, float, float]:
    """Compute spectral metrics: rolloff, effective nyquist, high freq ratio."""
    if audio.size == 0:
        return 0.0, 0.0, 0.0

    max_samples = min(audio.size, sr * 5)
    window = get_hann_window(max_samples)
    chunk = audio[:max_samples] * window
    spec = np.fft.rfft(chunk)
    power = np.abs(spec) ** 2
    freqs = get_rfftfreq(max_samples, sr)
    total_energy = float(np.sum(power)) + 1e-12
    cumulative = np.cumsum(power)
    rolloff_idx = int(np.searchsorted(cumulative, 0.85 * cumulative[-1]))
    rolloff_hz = float(freqs[min(rolloff_idx, len(freqs) - 1)])

    power_db = 10.0 * np.log10(power + 1e-12)
    threshold = power_db.max() - 60.0
    above = np.where(power_db >= threshold)[0]
    effective_nyquist_hz = float(freqs[above[-1]]) if above.size > 0 else 0.0

    high_freq_energy = float(np.sum(power[freqs >= 8000.0]))
    high_freq_ratio = high_freq_energy / total_energy

    return rolloff_hz, effective_nyquist_hz, high_freq_ratio

def classify_native_bucket(effective_nyquist_hz: float) -> str:
    """Classify audio bandwidth based on effective Nyquist frequency."""
    if effective_nyquist_hz < 8000:
        return "<16k"
    if effective_nyquist_hz < 12000:
        return "16-24k"
    if effective_nyquist_hz < 16000:
        return "24-32k"
    if effective_nyquist_hz < 22050:
        return "32-44k"
    return ">=44k"

def compute_audio_metrics(audio: np.ndarray, sr: int) -> dict[str, float]:
    """Compute all audio metrics for a segment."""
    audio = audio.astype(np.float32, copy=False)
    rms = float(np.sqrt(np.mean(audio ** 2) + 1e-12))
    loudness_rms_db = 20.0 * math.log10(rms + 1e-12)
    clipping_fraction = float(np.mean(np.abs(audio) > 0.99))
    dc_offset = float(np.mean(audio))
    noise_floor = float(np.percentile(np.abs(audio), 10))
    snr_proxy = 20.0 * math.log10((rms + 1e-9) / (noise_floor + 1e-9))

    loudness_lufs = -100.0
    if pyln is not None:
        try:
            meter = get_lufs_meter(sr)
            if meter is not None:
                loudness_lufs = float(meter.integrated_loudness(audio))
        except Exception:
            loudness_lufs = -100.0

    rolloff_hz, effective_nyquist_hz, high_freq_ratio = compute_spectral_metrics(audio, sr)

    return {
        "loudness_lufs": loudness_lufs,
        "loudness_rms_db": loudness_rms_db,
        "clipping_fraction": clipping_fraction,
        "dc_offset": dc_offset,
        "snr_proxy": snr_proxy,
        "spectral_rolloff_hz": rolloff_hz,
        "effective_nyquist_hz": effective_nyquist_hz,
        "high_freq_energy_ratio": high_freq_ratio,
    }

def compute_quality_score(metrics: dict[str, float], has_music: bool) -> float:
    """Compute overall quality score."""
    score = (
        0.3 * (1.0 - metrics["clipping_fraction"])
        + 0.2 * normalize(metrics["snr_proxy"], 10, 40)
        + 0.2 * (1.0 if not has_music else 0.5)
        + 0.2 * normalize(metrics["loudness_lufs"], -40, -14)
        + 0.1 * (1.0 if abs(metrics["dc_offset"]) < 0.01 else 0.5)
    )
    return max(0.0, min(1.0, score))

def resample_audio(audio: np.ndarray, src_sr: int, dst_sr: int) -> np.ndarray:
    """Resample audio to target sample rate."""
    if src_sr == dst_sr:
        return audio
    gcd = math.gcd(src_sr, dst_sr)
    up = dst_sr // gcd
    down = src_sr // gcd
    return signal.resample_poly(audio, up, down).astype(np.float32, copy=False)

def encode_audio(audio: np.ndarray, sr: int, output_format: str) -> bytes:
    """Encode audio to bytes."""
    buf = io.BytesIO()
    if output_format == "wav":
        sf.write(buf, audio, sr, format="WAV", subtype="PCM_16")
    else:
        sf.write(buf, audio, sr, format="FLAC", subtype="PCM_16")
    return buf.getvalue()

def duration_bucket(duration_sec: float) -> Optional[tuple[str, float]]:
    """Get duration bucket and weight."""
    if duration_sec < 3.0:
        return None
    if duration_sec < 6.0:
        return "3-6s", 0.3
    if duration_sec < 12.0:
        return "6-12s", 1.0
    if duration_sec < 24.0:
        return "12-24s", 0.8
    if duration_sec <= 60.0:
        return "24-60s", 0.5
    return None

def compute_frame_rms(
    audio: np.ndarray, sr: int, frame_sec: float, hop_sec: float
) -> tuple[np.ndarray, np.ndarray]:
    """Compute frame-level RMS for silence detection."""
    frame_len = max(1, int(round(frame_sec * sr)))
    hop_len = max(1, int(round(hop_sec * sr)))
    if audio.size < frame_len:
        return np.array([], dtype=np.float32), np.array([], dtype=np.float32)

    if not audio.flags["C_CONTIGUOUS"]:
        audio = np.ascontiguousarray(audio)

    num_frames = 1 + (audio.size - frame_len) // hop_len
    if num_frames <= 0:
        return np.array([], dtype=np.float32), np.array([], dtype=np.float32)

    stride = audio.strides[0]
    frames = np.lib.stride_tricks.as_strided(
        audio,
        shape=(num_frames, frame_len),
        strides=(hop_len * stride, stride),
    )
    rms = np.sqrt(np.mean(frames * frames, axis=1) + 1e-12)
    times = (np.arange(num_frames, dtype=np.float32) * hop_len + (frame_len / 2)) / sr
    return times.astype(np.float32), rms.astype(np.float32)

def find_silence_boundary(
    times: np.ndarray,
    rms: np.ndarray,
    search_start: float,
    search_end: float,
    threshold_rms: float,
    median_db: float,
    db_drop: float,
) -> Optional[float]:
    """Find optimal silence boundary for chunking."""
    if times.size == 0:
        return None
    mask = (times >= search_start) & (times <= search_end)
    if not np.any(mask):
        return None
    subset = rms[mask]
    min_idx = int(np.argmin(subset))
    min_rms = float(subset[min_idx])
    min_time = float(times[mask][min_idx])
    min_db = 20.0 * math.log10(min_rms + 1e-12)
    if min_rms <= threshold_rms or (median_db - min_db) >= db_drop:
        return min_time
    return None

def make_chunk_windows(
    audio: np.ndarray,
    sr: int,
    cfg: ChunkConfig,
    silence_cfg: SilenceConfig,
    rng: random.Random,
) -> list[tuple[float, float]]:
    """Generate chunk windows for a segment."""
    duration_sec = float(audio.shape[0] / sr)
    if not cfg.enabled or duration_sec < cfg.threshold_sec:
        return [(0.0, duration_sec)]

    if not silence_cfg.enabled:
        windows: list[tuple[float, float]] = []
        offset = 0.0
        while duration_sec - offset >= cfg.min_sec:
            remaining = duration_sec - offset
            if remaining <= cfg.max_sec:
                length = remaining
            else:
                length = rng.uniform(cfg.min_sec, cfg.max_sec)
                if remaining - length < cfg.min_sec:
                    length = remaining
            windows.append((offset, length))
            offset += length
        return windows

    times, rms = compute_frame_rms(
        audio, sr, silence_cfg.frame_sec, silence_cfg.hop_sec
    )
    if times.size == 0:
        return [(0.0, duration_sec)]

    threshold_rms = float(np.quantile(rms, silence_cfg.percentile / 100.0))
    median_db = 20.0 * math.log10(float(np.median(rms)) + 1e-12)

    windows: list[tuple[float, float]] = []
    offset = 0.0
    while duration_sec - offset > cfg.max_sec:
        remaining = duration_sec - offset
        latest_cut = duration_sec - cfg.min_sec
        if offset + cfg.min_sec >= latest_cut:
            break
        target_len = rng.uniform(cfg.min_sec, cfg.max_sec)
        if remaining - target_len < cfg.min_sec:
            target_len = remaining - cfg.min_sec
        target_end = offset + target_len
        search_low = max(offset + cfg.min_sec, target_end - silence_cfg.search_sec)
        search_high = min(offset + cfg.max_sec, target_end + silence_cfg.search_sec, latest_cut)
        if search_high <= search_low:
            search_low = offset + cfg.min_sec
            search_high = min(offset + cfg.max_sec, latest_cut)
        boundary = find_silence_boundary(
            times,
            rms,
            search_low,
            search_high,
            threshold_rms,
            median_db,
            silence_cfg.db_drop,
        )
        if boundary is None:
            if not windows:
                return [(0.0, duration_sec)]
            windows.append((offset, duration_sec - offset))
            return windows
        windows.append((offset, boundary - offset))
        offset = boundary

    windows.append((offset, duration_sec - offset))
    return windows

def make_segment_seed(base_seed: int, video_id: str, speaker: str, index: int) -> int:
    """Generate deterministic seed for a segment."""
    key = f"{video_id}-{speaker}-{index}-{base_seed}"
    return zlib.adler32(key.encode("utf-8")) & 0xFFFFFFFF

def parse_segment_filename(filename: str) -> Optional[tuple[str, float]]:
    """Parse segment filename to extract speaker and start time."""
    base = filename.split("/")[-1]
    if not base.endswith(".flac"):
        return None

    base = base[:-5]
    parts = base.split("_")
    if len(parts) < 3:
        return None

    speaker = "_".join(parts[:-2])
    time_range = parts[-1]
    if "-" not in time_range:
        return None

    try:
        start_str = time_range.split("-")[0]
        start_time = float(start_str)
    except (ValueError, IndexError):
        return None

    return speaker, start_time

def process_segment(
    video_id: str,
    language: str,
    segment_bytes: bytes,
    segment_meta: dict[str, Any],
    speaker: str,
    index: int,
    chunk_cfg: ChunkConfig,
    silence_cfg: SilenceConfig,
    rng_seed: int,
    output_format: str,
) -> tuple[list[SegmentOutput], dict[str, int]]:
    """Process a single segment and return outputs."""
    stats = {
        "dropped_too_short": 0,
        "dropped_too_long": 0,
        "dropped_clipped": 0,
        "chunks_emitted": 0,
        "segments_chunked": 0,
    }
    outputs: list[SegmentOutput] = []

    try:
        audio, sr = sf.read(io.BytesIO(segment_bytes), dtype="float32")
    except Exception:
        return outputs, stats

    if audio.ndim > 1:
        audio = np.mean(audio, axis=1)

    rng = random.Random(rng_seed)
    windows = make_chunk_windows(audio, sr, chunk_cfg, silence_cfg, rng)

    music_stats = segment_meta.get("music_stats") or {}
    has_music = bool(music_stats.get("has_music", False))

    def emit_outputs(
        audio_slice: np.ndarray,
        metrics: dict[str, float],
        sample_id: str,
        segment_index: int,
        chunk_index: Optional[int],
        start: Optional[float],
        end: Optional[float],
        original_start: Optional[float],
        original_end: Optional[float],
        target_srs: list[int],
    ) -> None:
        slice_duration = float(audio_slice.shape[0] / sr)
        bucket = duration_bucket(slice_duration)
        if bucket is None:
            if slice_duration < 3.0:
                stats["dropped_too_short"] += 1
            else:
                stats["dropped_too_long"] += 1
            return
        if metrics["clipping_fraction"] > 0.1:
            stats["dropped_clipped"] += 1
            return

        duration_bucket_name, base_weight = bucket
        sampling_weight = base_weight * (0.5 if has_music else 1.0)
        quality_score = compute_quality_score(metrics, has_music)
        effective_nyquist = metrics["effective_nyquist_hz"]

        base_metadata = {
            "id": sample_id,
            "video_id": video_id,
            "language": language,
            "segment_index": segment_index,
            "chunk_index": chunk_index,
            "start": start,
            "end": end,
            "original_start": original_start,
            "original_end": original_end,
            "duration_sec": slice_duration,
            "native_sr": sr,
            "effective_nyquist_hz": effective_nyquist,
            "spectral_rolloff_hz": metrics["spectral_rolloff_hz"],
            "high_freq_energy_ratio": metrics["high_freq_energy_ratio"],
            "is_truly_wideband": bool(effective_nyquist >= 12000),
            "native_sr_bucket": classify_native_bucket(effective_nyquist),
            "loudness_lufs": metrics["loudness_lufs"],
            "loudness_rms_db": metrics["loudness_rms_db"],
            "clipping_fraction": metrics["clipping_fraction"],
            "snr_proxy": metrics["snr_proxy"],
            "quality_score": quality_score,
            "music_prob": float(music_stats.get("music_mean", 0.0)),
            "music_stats": music_stats,
            "speaker": speaker,
            "vad_speech_regions": [[0.0, slice_duration]],
            "duration_bucket": duration_bucket_name,
            "sampling_weight": sampling_weight,
        }

        for target_sr in target_srs:
            resampled = resample_audio(audio_slice, sr, target_sr)
            resampled = np.clip(resampled, -1.0, 1.0)
            audio_bytes = encode_audio(resampled, target_sr, output_format)
            meta = dict(base_metadata)
            meta["sample_rate"] = target_sr
            meta["resample_target_sr"] = target_sr
            outputs.append(SegmentOutput(target_sr, audio_bytes, meta))

        if target_srs == [16000]:
            stats["chunks_emitted"] += 1

    sample_id_base = f"{video_id}__{speaker}_{index:04d}"
    use_chunk_suffix = len(windows) > 1
    if use_chunk_suffix:
        stats["segments_chunked"] += 1

    segment_start = segment_meta.get("start")
    segment_end = segment_meta.get("end")
    original_start = segment_meta.get("original_start")
    original_end = segment_meta.get("original_end")

    # Emit 16k outputs for all chunks
    for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
        start_idx = int(round(offset_sec * sr))
        end_idx = int(round((offset_sec + chunk_len_sec) * sr))
        audio_slice = audio[start_idx:end_idx]
        if audio_slice.size == 0:
            continue
        slice_duration = float(audio_slice.shape[0] / sr)
        slice_metrics = compute_audio_metrics(audio_slice, sr)

        chunk_start = float(segment_start) + offset_sec if segment_start is not None else None
        chunk_end = chunk_start + slice_duration if chunk_start is not None else None
        chunk_original_start = (
            float(original_start) + offset_sec if original_start is not None else None
        )
        chunk_original_end = (
            chunk_original_start + slice_duration
            if chunk_original_start is not None
            else None
        )

        sample_id = sample_id_base
        chunk_index = None
        if use_chunk_suffix:
            sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
            chunk_index = chunk_idx

        emit_outputs(
            audio_slice,
            slice_metrics,
            sample_id,
            index,
            chunk_index,
            chunk_start,
            chunk_end,
            chunk_original_start,
            chunk_original_end,
            [16000],
        )

    # High-rate buckets for full segment
    full_metrics = compute_audio_metrics(audio, sr)
    effective_nyquist = full_metrics["effective_nyquist_hz"]
    targets_high: list[int] = []
    if effective_nyquist >= 8000:
        targets_high.append(24000)
    if effective_nyquist >= 12000:
        targets_high.append(32000)
    if effective_nyquist >= 16000:
        targets_high.append(44100)

    if targets_high:
        if chunk_cfg.all_buckets and use_chunk_suffix:
            for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
                start_idx = int(round(offset_sec * sr))
                end_idx = int(round((offset_sec + chunk_len_sec) * sr))
                audio_slice = audio[start_idx:end_idx]
                if audio_slice.size == 0:
                    continue
                slice_duration = float(audio_slice.shape[0] / sr)
                slice_metrics = compute_audio_metrics(audio_slice, sr)
                slice_effective_nyquist = slice_metrics["effective_nyquist_hz"]
                slice_targets: list[int] = []
                if slice_effective_nyquist >= 8000:
                    slice_targets.append(24000)
                if slice_effective_nyquist >= 12000:
                    slice_targets.append(32000)
                if slice_effective_nyquist >= 16000:
                    slice_targets.append(44100)
                if not slice_targets:
                    continue

                chunk_start = (
                    float(segment_start) + offset_sec if segment_start is not None else None
                )
                chunk_end = (
                    chunk_start + slice_duration if chunk_start is not None else None
                )
                chunk_original_start = (
                    float(original_start) + offset_sec if original_start is not None else None
                )
                chunk_original_end = (
                    chunk_original_start + slice_duration
                    if chunk_original_start is not None
                    else None
                )

                sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
                emit_outputs(
                    audio_slice,
                    slice_metrics,
                    sample_id,
                    index,
                    chunk_idx,
                    chunk_start,
                    chunk_end,
                    chunk_original_start,
                    chunk_original_end,
                    slice_targets,
                )
        else:
            emit_outputs(
                audio,
                full_metrics,
                sample_id_base,
                index,
                None,
                segment_start,
                segment_end,
                original_start,
                original_end,
                targets_high,
            )

    return outputs, stats

class ShardWriter:
    """Write samples to tar + jsonl shards."""

    def __init__(
        self, output_dir: Path, prefix: str, target_size_bytes: int, audio_ext: str
    ) -> None:
        self.output_dir = output_dir
        self.prefix = prefix
        self.target_size_bytes = target_size_bytes
        self.audio_ext = audio_ext
        self.shard_index = 0
        self.current_tar: Optional[tarfile.TarFile] = None
        self.current_jsonl = None
        self.current_size = 0
        self.current_count = 0
        self.current_name = ""
        self.output_dir.mkdir(parents=True, exist_ok=True)

    def _open_new_shard(self) -> None:
        if self.current_tar:
            self.close()
        shard_name = f"{self.prefix}-{self.shard_index:06d}"
        tar_path = self.output_dir / f"{shard_name}.tar"
        jsonl_path = self.output_dir / f"{shard_name}.jsonl"
        self.current_tar = tarfile.open(tar_path, "w")
        self.current_jsonl = jsonl_path.open("w", encoding="utf-8")
        self.current_size = 0
        self.current_count = 0
        self.current_name = shard_name
        self.shard_index += 1

    def add_sample(self, sample_id: str, audio_bytes: bytes, metadata: dict[str, Any]) -> str:
        if self.current_tar is None:
            self._open_new_shard()
        if (
            self.target_size_bytes > 0
            and self.current_count > 0
            and self.current_size + len(audio_bytes) > self.target_size_bytes
        ):
            self._open_new_shard()

        assert self.current_tar is not None
        assert self.current_jsonl is not None

        arcname = f"{sample_id}.{self.audio_ext}"
        info = tarfile.TarInfo(name=arcname)
        info.size = len(audio_bytes)
        info.mtime = int(time.time())
        self.current_tar.addfile(info, io.BytesIO(audio_bytes))
        self.current_jsonl.write(json.dumps(metadata) + "\n")

        self.current_size += len(audio_bytes)
        self.current_count += 1
        return self.current_name

    def close(self) -> None:
        if self.current_tar:
            self.current_tar.close()
        if self.current_jsonl:
            self.current_jsonl.close()
        self.current_tar = None
        self.current_jsonl = None

class VideoProcessor:
    """Process videos and write to shards."""

    def __init__(
        self,
        output_dir: Path,
        language: str,
        shard_prefix: str,
        chunk_cfg: ChunkConfig,
        silence_cfg: SilenceConfig,
        output_format: str = "flac",
        target_shard_size_mb: int = 5120,
        segment_workers: int = 8,
        seed: int = 17,
        val_ratio: float = 0.01,
        on_progress: Optional[Callable[[int, int], None]] = None,
    ):
        self.output_dir = output_dir
        self.language = language
        self.shard_prefix = shard_prefix
        self.chunk_cfg = chunk_cfg
        self.silence_cfg = silence_cfg
        self.output_format = output_format
        self.seed = seed
        self.val_ratio = val_ratio
        self.segment_workers = segment_workers
        self.on_progress = on_progress

        audio_ext = "wav" if output_format == "wav" else "flac"
        target_bytes = max(0, target_shard_size_mb) * 1024 * 1024

        self.writers = {
            16000: ShardWriter(
                output_dir / f"{language}_16", shard_prefix, target_bytes, audio_ext
            ),
            24000: ShardWriter(
                output_dir / f"{language}_24", shard_prefix, target_bytes, audio_ext
            ),
            32000: ShardWriter(
                output_dir / f"{language}_32", shard_prefix, target_bytes, audio_ext
            ),
            44100: ShardWriter(
                output_dir / f"{language}_44", shard_prefix, target_bytes, audio_ext
            ),
        }

        # Manifests
        manifest_dir = output_dir / "manifests"
        manifest_dir.mkdir(parents=True, exist_ok=True)
        self.train_manifest = (manifest_dir / f"{language}_train_{shard_prefix}.jsonl").open(
            "w", encoding="utf-8"
        )
        self.val_manifest = (manifest_dir / f"{language}_val_{shard_prefix}.jsonl").open(
            "w", encoding="utf-8"
        )

        self.stats = ProcessingStats()
        self.audio_ext = audio_ext

    def process_videos(
        self,
        download_paths: dict[str, Path],
        val_ids: Optional[set[str]] = None,
    ) -> ProcessingStats:
        """Process all videos and write to shards."""
        if val_ids is None:
            # Determine val set
            video_ids = list(download_paths.keys())
            random.seed(self.seed)
            val_count = max(1, int(len(video_ids) * self.val_ratio)) if len(video_ids) > 1 else 0
            val_ids = set(random.sample(video_ids, val_count)) if val_count > 0 else set()

        max_inflight = max(128, self.segment_workers * 2)
        pending: set[Any] = set()
        future_info: dict[Any, bool] = {}

        def handle_future(future) -> None:
            is_val = future_info.pop(future, False)
            outputs, seg_stats = future.result()
            self.stats.dropped_too_short += seg_stats["dropped_too_short"]
            self.stats.dropped_too_long += seg_stats["dropped_too_long"]
            self.stats.dropped_clipped += seg_stats["dropped_clipped"]
            self.stats.segments_chunked += seg_stats.get("segments_chunked", 0)
            self.stats.chunks_emitted += seg_stats.get("chunks_emitted", 0)

            for out in outputs:
                self.stats.segments_kept[out.sample_rate] += 1
                if out.sample_rate == 16000:
                    bucket = out.metadata.get("native_sr_bucket")
                    if bucket:
                        self.stats.native_sr_buckets[bucket] = (
                            self.stats.native_sr_buckets.get(bucket, 0) + 1
                        )
                writer = self.writers[out.sample_rate]
                shard_name = writer.add_sample(
                    out.metadata["id"], out.audio_bytes, out.metadata
                )
                manifest_entry = dict(out.metadata)
                manifest_entry["shard"] = f"{shard_name}.tar"
                manifest_entry["audio"] = f"{out.metadata['id']}.{self.audio_ext}"
                line = json.dumps(manifest_entry) + "\n"
                if is_val:
                    self.val_manifest.write(line)
                else:
                    self.train_manifest.write(line)

        def drain_completed() -> None:
            nonlocal pending
            if not pending:
                return
            done, pending = wait(pending, return_when=FIRST_COMPLETED)
            for future in done:
                handle_future(future)

        total_videos = len(download_paths)
        processed_videos = 0

        with ThreadPoolExecutor(max_workers=self.segment_workers) as executor:
            for video_id, tar_path in download_paths.items():
                self.stats.videos += 1
                is_val = video_id in val_ids

                try:
                    tf = tarfile.open(tar_path, "r")
                except Exception as e:
                    continue

                with tf:
                    try:
                        meta = json.loads(
                            tf.extractfile(f"{video_id}/metadata.json").read()
                        )
                    except Exception:
                        continue

                    usable_segments = [
                        s for s in meta.get("segments", []) if s.get("status") == "usable"
                    ]
                    segment_map = {
                        (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
                        for seg in usable_segments
                    }

                    members = [m for m in tf.getmembers() if m.name.endswith(".flac")]
                    self.stats.segments_total += len(members)

                    for member in members:
                        parsed = parse_segment_filename(member.name)
                        if not parsed:
                            continue
                        speaker, start_time = parsed
                        seg_meta = segment_map.get((speaker, round(start_time, 2)))
                        if not seg_meta:
                            matches = [
                                s for s in usable_segments
                                if s.get("speaker") == speaker
                                and abs(s.get("start", 0.0) - start_time) < 0.1
                            ]
                            if not matches:
                                continue
                            seg_meta = matches[0]

                        while len(pending) >= max_inflight:
                            drain_completed()

                        segment_bytes = tf.extractfile(member).read()
                        filename_parts = member.name.split("_")
                        try:
                            index = int(filename_parts[-2])
                        except (ValueError, IndexError):
                            index = 0
                        rng_seed = make_segment_seed(self.seed, video_id, speaker, index)

                        future = executor.submit(
                            process_segment,
                            video_id,
                            self.language,
                            segment_bytes,
                            seg_meta,
                            speaker,
                            index,
                            self.chunk_cfg,
                            self.silence_cfg,
                            rng_seed,
                            self.output_format,
                        )
                        future_info[future] = is_val
                        pending.add(future)

                processed_videos += 1
                if self.on_progress:
                    self.on_progress(processed_videos, total_videos)

            while pending:
                drain_completed()

        return self.stats

    def close(self) -> None:
        """Close all writers and manifests."""
        for writer in self.writers.values():
            writer.close()
        self.train_manifest.close()
        self.val_manifest.close()

    def save_stats(self) -> Path:
        """Save processing stats to JSON."""
        stats_dir = self.output_dir / "stats"
        stats_dir.mkdir(parents=True, exist_ok=True)
        stats_path = stats_dir / f"{self.language}_stats_{self.shard_prefix}.json"
        stats_path.write_text(json.dumps(self.stats.to_dict(), indent=2))
        return stats_path