Excellent! Now let me search for any additional dependencies and related modules to understand the complete picture:


Grep

import soundfile|from scipy|import pyloudnorm|np.fft|signal.resample


Bash

grep -r "def.*encode\|def.*decode\|def.*resample" /home/ubuntu/maya3_prepare/scripts/ 2>/dev/null | head -20

Search for audio encoding/decoding functions


/home/ubuntu/maya3_prepare/scripts/prepare_shards.py:def resample_audio(
/home/ubuntu/maya3_prepare/scripts/prepare_shards.py:def encode_audio(audio: np.ndarray, sr: int, output_format: str) -> bytes:


22:import soundfile as sf
25:from scipy import signal
29:    import pyloudnorm as pyln
112:            freqs = np.fft.rfftfreq(length, d=1.0 / sr)
273:        spec = np.fft.rfft(chunk)
364:        return signal.resample_poly(audio, up, down).astype(np.float32, copy=False)


Perfect! Now let me create a comprehensive analysis document by examining the key sections more carefully:


head -50 /home/ubuntu/maya3_prepare/scripts/prepare_shards.py | grep -E "import|from"

Get all imports at the top


     1→from __future__ import annotations
     2→
     3→import argparse
     4→import io
     5→import json
     6→import math
     7→import os
     8→import random
     9→import tarfile
    10→import threading
    11→import time
    12→import zlib
    13→from collections import defaultdict
    14→from contextlib import contextmanager
    15→from dataclasses import dataclass
    16→from pathlib import Path
    17→from typing import Any, Optional
    18→
    19→import boto3
    20→import numpy as np
    21→import psycopg
    22→import soundfile as sf
    23→from boto3.s3.transfer import TransferConfig
    24→from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
    25→from scipy import signal
    26→from tqdm import tqdm
    27→
    28→try:
    29→    import pyloudnorm as pyln
    30→except Exception:  # pragma: no cover - optional dependency
    31→    pyln = None
    32→
    33→ENV_PATH = Path(".env")
    34→LANGUAGES = [
    35→    "telugu",
    36→    "malayalam",
    37→    "english",
    38→    "hindi",
    39→    "punjabi",
    40→    "tamil",
    41→    "kannada",
    42→    "gujarati",
    43→    "bengali",
    44→    "odia",
    45→    "marathi",
    46→    "assamese",
    47→]
    48→
    49→# Global timing stats (thread-safe)
    50→_timing_lock = threading.Lock()
    51→_timing_stats: dict[str, list[float]] = defaultdict(list)
    52→_timing_enabled = False
    53→
    54→
    55→@contextmanager
    56→def timer(name: str):
    57→    """Context manager for timing code blocks."""
    58→    if not _timing_enabled:
    59→        yield
    60→        return
    61→    start = time.perf_counter()
    62→    try:
    63→        yield
    64→    finally:
    65→        elapsed = time.perf_counter() - start
    66→        with _timing_lock:
    67→            _timing_stats[name].append(elapsed)
    68→
    69→
    70→def print_timing_stats():
    71→    """Print aggregated timing statistics."""
    72→    if not _timing_stats:
    73→        return
    74→    print("\n=== Timing Statistics ===")
    75→    total_time = sum(sum(times) for times in _timing_stats.values())
    76→    for name, times in sorted(_timing_stats.items()):
    77→        count = len(times)
    78→        total = sum(times)
    79→        avg = total / count if count > 0 else 0
    80→        percent = (total / total_time * 100) if total_time > 0 else 0
    81→        print(f"{name:35s}: {count:6d} calls, {total:8.2f}s total, {avg*1000:8.2f}ms avg, {percent:5.1f}%")
    82→    print(f"{'TOTAL':35s}:              {total_time:8.2f}s")
    83→    print("=" * 80)
    84→
    85→
    86→_window_cache: dict[int, np.ndarray] = {}
    87→_freq_cache: dict[tuple[int, int], np.ndarray] = {}
    88→_cache_lock = threading.Lock()
    89→_lufs_local = threading.local()
    90→
    91→
    92→def get_hann_window(length: int) -> np.ndarray:
    93→    window = _window_cache.get(length)
    94→    if window is not None:
    95→        return window
    96→    with _cache_lock:
    97→        window = _window_cache.get(length)
    98→        if window is None:
    99→            window = np.hanning(length)
   100→            _window_cache[length] = window

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

from __future__ import annotations

import argparse
import io
import json
import math
import os
import random
import tarfile
import threading
import time
import zlib
from collections import defaultdict
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Optional

import boto3
import numpy as np
import psycopg
import soundfile as sf
from boto3.s3.transfer import TransferConfig
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
from scipy import signal
from tqdm import tqdm

try:
    import pyloudnorm as pyln
except Exception:  # pragma: no cover - optional dependency
    pyln = None

ENV_PATH = Path(".env")
LANGUAGES = [
    "telugu",
    "malayalam",
    "english",
    "hindi",
    "punjabi",
    "tamil",
    "kannada",
    "gujarati",
    "bengali",
    "odia",
    "marathi",
    "assamese",
]

# Global timing stats (thread-safe)
_timing_lock = threading.Lock()
_timing_stats: dict[str, list[float]] = defaultdict(list)
_timing_enabled = False

@contextmanager
def timer(name: str):
    """Context manager for timing code blocks."""
    if not _timing_enabled:
        yield
        return
    start = time.perf_counter()
    try:
        yield
    finally:
        elapsed = time.perf_counter() - start
        with _timing_lock:
            _timing_stats[name].append(elapsed)

def print_timing_stats():
    """Print aggregated timing statistics."""
    if not _timing_stats:
        return
    print("\n=== Timing Statistics ===")
    total_time = sum(sum(times) for times in _timing_stats.values())
    for name, times in sorted(_timing_stats.items()):
        count = len(times)
        total = sum(times)
        avg = total / count if count > 0 else 0
        percent = (total / total_time * 100) if total_time > 0 else 0
        print(f"{name:35s}: {count:6d} calls, {total:8.2f}s total, {avg*1000:8.2f}ms avg, {percent:5.1f}%")
    print(f"{'TOTAL':35s}:              {total_time:8.2f}s")
    print("=" * 80)

_window_cache: dict[int, np.ndarray] = {}
_freq_cache: dict[tuple[int, int], np.ndarray] = {}
_cache_lock = threading.Lock()
_lufs_local = threading.local()

def get_hann_window(length: int) -> np.ndarray:
    window = _window_cache.get(length)
    if window is not None:
        return window
    with _cache_lock:
        window = _window_cache.get(length)
        if window is None:
            window = np.hanning(length)
            _window_cache[length] = window


from __future__ import annotations
import argparse
import io
import json
import math
import os
import random
import tarfile
import threading
import time
import zlib
from collections import defaultdict
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Optional
import boto3
import numpy as np
import psycopg
import soundfile as sf
from boto3.s3.transfer import TransferConfig
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
from scipy import signal
from tqdm import tqdm
    import pyloudnorm as pyln


Now let me examine the key audio processing functions in detail to understand the implementation:


   266→def compute_spectral_metrics(audio: np.ndarray, sr: int) -> tuple[float, float, float]:
   267→    if audio.size == 0:
   268→        return 0.0, 0.0, 0.0
   269→    with timer("cpu_spectral_compute"):
   270→        max_samples = min(audio.size, sr * 5)
   271→        window = get_hann_window(max_samples)
   272→        chunk = audio[:max_samples] * window
   273→        spec = np.fft.rfft(chunk)
   274→        power = np.abs(spec) ** 2
   275→        freqs = get_rfftfreq(max_samples, sr)
   276→        total_energy = float(np.sum(power)) + 1e-12
   277→        cumulative = np.cumsum(power)
   278→        rolloff_idx = int(np.searchsorted(cumulative, 0.85 * cumulative[-1]))
   279→        rolloff_hz = float(freqs[min(rolloff_idx, len(freqs) - 1)])
   280→
   281→        power_db = 10.0 * np.log10(power + 1e-12)
   282→        threshold = power_db.max() - 60.0
   283→        above = np.where(power_db >= threshold)[0]
   284→        effective_nyquist_hz = float(freqs[above[-1]]) if above.size > 0 else 0.0
   285→
   286→        high_freq_energy = float(np.sum(power[freqs >= 8000.0]))
   287→        high_freq_ratio = high_freq_energy / total_energy
   288→    return rolloff_hz, effective_nyquist_hz, high_freq_ratio
   289→
   290→
   291→def classify_native_bucket(effective_nyquist_hz: float) -> str:
   292→    """Classify audio bandwidth based on effective Nyquist frequency.
   293→
   294→    These buckets represent actual audio bandwidth, not container sample rates.
   295→    Thresholds are based on observed distribution in the dataset.
   296→    """
   297→    if effective_nyquist_hz < 8000:
   298→        return "<16k"
   299→    if effective_nyquist_hz < 12000:
   300→        return "16-24k"
   301→    if effective_nyquist_hz < 16000:
   302→        return "24-32k"
   303→    if effective_nyquist_hz < 22050:
   304→        return "32-44k"
   305→    return ">=44k"
   306→
   307→
   308→def compute_audio_metrics(audio: np.ndarray, sr: int) -> dict[str, float]:
   309→    with timer("audio_metrics_basic"):
   310→        audio = audio.astype(np.float32, copy=False)
   311→        rms = float(np.sqrt(np.mean(audio ** 2) + 1e-12))
   312→        loudness_rms_db = 20.0 * math.log10(rms + 1e-12)
   313→        clipping_fraction = float(np.mean(np.abs(audio) > 0.99))
   314→        dc_offset = float(np.mean(audio))
   315→        noise_floor = float(np.percentile(np.abs(audio), 10))
   316→        snr_proxy = 20.0 * math.log10((rms + 1e-9) / (noise_floor + 1e-9))
   317→
   318→    with timer("audio_metrics_lufs"):
   319→        loudness_lufs = -100.0
   320→        if pyln is not None:
   321→            try:
   322→                meter = get_lufs_meter(sr)
   323→                if meter is not None:
   324→                    loudness_lufs = float(meter.integrated_loudness(audio))
   325→            except Exception:
   326→                loudness_lufs = -100.0
   327→
   328→    rolloff_hz, effective_nyquist_hz, high_freq_ratio = compute_spectral_metrics(audio, sr)
   329→
   330→    return {
   331→        "loudness_lufs": loudness_lufs,
   332→        "loudness_rms_db": loudness_rms_db,
   333→        "clipping_fraction": clipping_fraction,
   334→        "dc_offset": dc_offset,
   335→        "snr_proxy": snr_proxy,
   336→        "spectral_rolloff_hz": rolloff_hz,
   337→        "effective_nyquist_hz": effective_nyquist_hz,
   338→        "high_freq_energy_ratio": high_freq_ratio,
   339→    }
   340→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

def compute_spectral_metrics(audio: np.ndarray, sr: int) -> tuple[float, float, float]:
    if audio.size == 0:
        return 0.0, 0.0, 0.0
    with timer("cpu_spectral_compute"):
        max_samples = min(audio.size, sr * 5)
        window = get_hann_window(max_samples)
        chunk = audio[:max_samples] * window
        spec = np.fft.rfft(chunk)
        power = np.abs(spec) ** 2
        freqs = get_rfftfreq(max_samples, sr)
        total_energy = float(np.sum(power)) + 1e-12
        cumulative = np.cumsum(power)
        rolloff_idx = int(np.searchsorted(cumulative, 0.85 * cumulative[-1]))
        rolloff_hz = float(freqs[min(rolloff_idx, len(freqs) - 1)])

        power_db = 10.0 * np.log10(power + 1e-12)
        threshold = power_db.max() - 60.0
        above = np.where(power_db >= threshold)[0]
        effective_nyquist_hz = float(freqs[above[-1]]) if above.size > 0 else 0.0

        high_freq_energy = float(np.sum(power[freqs >= 8000.0]))
        high_freq_ratio = high_freq_energy / total_energy
    return rolloff_hz, effective_nyquist_hz, high_freq_ratio

def classify_native_bucket(effective_nyquist_hz: float) -> str:
    """Classify audio bandwidth based on effective Nyquist frequency.

    These buckets represent actual audio bandwidth, not container sample rates.
    Thresholds are based on observed distribution in the dataset.
    """
    if effective_nyquist_hz < 8000:
        return "<16k"
    if effective_nyquist_hz < 12000:
        return "16-24k"
    if effective_nyquist_hz < 16000:
        return "24-32k"
    if effective_nyquist_hz < 22050:
        return "32-44k"
    return ">=44k"

def compute_audio_metrics(audio: np.ndarray, sr: int) -> dict[str, float]:
    with timer("audio_metrics_basic"):
        audio = audio.astype(np.float32, copy=False)
        rms = float(np.sqrt(np.mean(audio ** 2) + 1e-12))
        loudness_rms_db = 20.0 * math.log10(rms + 1e-12)
        clipping_fraction = float(np.mean(np.abs(audio) > 0.99))
        dc_offset = float(np.mean(audio))
        noise_floor = float(np.percentile(np.abs(audio), 10))
        snr_proxy = 20.0 * math.log10((rms + 1e-9) / (noise_floor + 1e-9))

    with timer("audio_metrics_lufs"):
        loudness_lufs = -100.0
        if pyln is not None:
            try:
                meter = get_lufs_meter(sr)
                if meter is not None:
                    loudness_lufs = float(meter.integrated_loudness(audio))
            except Exception:
                loudness_lufs = -100.0

    rolloff_hz, effective_nyquist_hz, high_freq_ratio = compute_spectral_metrics(audio, sr)

    return {
        "loudness_lufs": loudness_lufs,
        "loudness_rms_db": loudness_rms_db,
        "clipping_fraction": clipping_fraction,
        "dc_offset": dc_offset,
        "snr_proxy": snr_proxy,
        "spectral_rolloff_hz": rolloff_hz,
        "effective_nyquist_hz": effective_nyquist_hz,
        "high_freq_energy_ratio": high_freq_ratio,
    }


   308→def compute_audio_metrics(audio: np.ndarray, sr: int) -> dict[str, float]:
   309→    with timer("audio_metrics_basic"):
   310→        audio = audio.astype(np.float32, copy=False)
   311→        rms = float(np.sqrt(np.mean(audio ** 2) + 1e-12))
   312→        loudness_rms_db = 20.0 * math.log10(rms + 1e-12)
   313→        clipping_fraction = float(np.mean(np.abs(audio) > 0.99))
   314→        dc_offset = float(np.mean(audio))
   315→        noise_floor = float(np.percentile(np.abs(audio), 10))
   316→        snr_proxy = 20.0 * math.log10((rms + 1e-9) / (noise_floor + 1e-9))
   317→
   318→    with timer("audio_metrics_lufs"):
   319→        loudness_lufs = -100.0
   320→        if pyln is not None:
   321→            try:
   322→                meter = get_lufs_meter(sr)
   323→                if meter is not None:
   324→                    loudness_lufs = float(meter.integrated_loudness(audio))
   325→            except Exception:
   326→                loudness_lufs = -100.0
   327→
   328→    rolloff_hz, effective_nyquist_hz, high_freq_ratio = compute_spectral_metrics(audio, sr)
   329→
   330→    return {
   331→        "loudness_lufs": loudness_lufs,
   332→        "loudness_rms_db": loudness_rms_db,
   333→        "clipping_fraction": clipping_fraction,
   334→        "dc_offset": dc_offset,
   335→        "snr_proxy": snr_proxy,
   336→        "spectral_rolloff_hz": rolloff_hz,
   337→        "effective_nyquist_hz": effective_nyquist_hz,
   338→        "high_freq_energy_ratio": high_freq_ratio,
   339→    }
   340→
   341→
   342→def compute_quality_score(metrics: dict[str, float], has_music: bool) -> float:

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

def compute_audio_metrics(audio: np.ndarray, sr: int) -> dict[str, float]:
    with timer("audio_metrics_basic"):
        audio = audio.astype(np.float32, copy=False)
        rms = float(np.sqrt(np.mean(audio ** 2) + 1e-12))
        loudness_rms_db = 20.0 * math.log10(rms + 1e-12)
        clipping_fraction = float(np.mean(np.abs(audio) > 0.99))
        dc_offset = float(np.mean(audio))
        noise_floor = float(np.percentile(np.abs(audio), 10))
        snr_proxy = 20.0 * math.log10((rms + 1e-9) / (noise_floor + 1e-9))

    with timer("audio_metrics_lufs"):
        loudness_lufs = -100.0
        if pyln is not None:
            try:
                meter = get_lufs_meter(sr)
                if meter is not None:
                    loudness_lufs = float(meter.integrated_loudness(audio))
            except Exception:
                loudness_lufs = -100.0

    rolloff_hz, effective_nyquist_hz, high_freq_ratio = compute_spectral_metrics(audio, sr)

    return {
        "loudness_lufs": loudness_lufs,
        "loudness_rms_db": loudness_rms_db,
        "clipping_fraction": clipping_fraction,
        "dc_offset": dc_offset,
        "snr_proxy": snr_proxy,
        "spectral_rolloff_hz": rolloff_hz,
        "effective_nyquist_hz": effective_nyquist_hz,
        "high_freq_energy_ratio": high_freq_ratio,
    }

def compute_quality_score(metrics: dict[str, float], has_music: bool) -> float:


   353→def resample_audio(
   354→    audio: np.ndarray,
   355→    src_sr: int,
   356→    dst_sr: int,
   357→) -> np.ndarray:
   358→    if src_sr == dst_sr:
   359→        return audio
   360→    with timer("cpu_resample"):
   361→        gcd = math.gcd(src_sr, dst_sr)
   362→        up = dst_sr // gcd
   363→        down = src_sr // gcd
   364→        return signal.resample_poly(audio, up, down).astype(np.float32, copy=False)
   365→
   366→
   367→def encode_audio(audio: np.ndarray, sr: int, output_format: str) -> bytes:
   368→    with timer("audio_encode"):
   369→        buf = io.BytesIO()
   370→        if output_format == "wav":
   371→            sf.write(buf, audio, sr, format="WAV", subtype="PCM_16")
   372→        else:
   373→            sf.write(buf, audio, sr, format="FLAC", subtype="PCM_16")
   374→        return buf.getvalue()
   375→
   376→
   377→def duration_bucket(duration_sec: float) -> Optional[tuple[str, float]]:

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

def resample_audio(
    audio: np.ndarray,
    src_sr: int,
    dst_sr: int,
) -> np.ndarray:
    if src_sr == dst_sr:
        return audio
    with timer("cpu_resample"):
        gcd = math.gcd(src_sr, dst_sr)
        up = dst_sr // gcd
        down = src_sr // gcd
        return signal.resample_poly(audio, up, down).astype(np.float32, copy=False)

def encode_audio(audio: np.ndarray, sr: int, output_format: str) -> bytes:
    with timer("audio_encode"):
        buf = io.BytesIO()
        if output_format == "wav":
            sf.write(buf, audio, sr, format="WAV", subtype="PCM_16")
        else:
            sf.write(buf, audio, sr, format="FLAC", subtype="PCM_16")
        return buf.getvalue()

def duration_bucket(duration_sec: float) -> Optional[tuple[str, float]]:


   555→def process_segment(
   556→    video_id: str,
   557→    language: str,
   558→    segment_bytes: bytes,
   559→    segment_meta: dict[str, Any],
   560→    speaker: str,
   561→    index: int,
   562→    chunk_cfg: ChunkConfig,
   563→    silence_cfg: SilenceConfig,
   564→    rng_seed: int,
   565→    output_format: str,
   566→) -> tuple[list[SegmentOutput], dict[str, int]]:
   567→    with timer("segment_total"):
   568→        stats = {
   569→            "dropped_too_short": 0,
   570→            "dropped_too_long": 0,
   571→            "dropped_clipped": 0,
   572→            "chunks_emitted": 0,
   573→            "segments_chunked": 0,
   574→        }
   575→        outputs: list[SegmentOutput] = []
   576→
   577→        with timer("flac_decode"):
   578→            try:
   579→                audio, sr = sf.read(io.BytesIO(segment_bytes), dtype="float32")
   580→            except Exception:
   581→                return outputs, stats
   582→
   583→        if audio.ndim > 1:
   584→            audio = np.mean(audio, axis=1)
   585→
   586→        rng = random.Random(rng_seed)
   587→        windows = make_chunk_windows(audio, sr, chunk_cfg, silence_cfg, rng)
   588→
   589→        music_stats = segment_meta.get("music_stats") or {}
   590→        has_music = bool(music_stats.get("has_music", False))
   591→
   592→        def emit_outputs(
   593→            audio_slice: np.ndarray,
   594→            metrics: dict[str, float],
   595→            sample_id: str,
   596→            segment_index: int,
   597→            chunk_index: Optional[int],
   598→            start: Optional[float],
   599→            end: Optional[float],
   600→            original_start: Optional[float],
   601→            original_end: Optional[float],
   602→            target_srs: list[int],
   603→        ) -> None:
   604→            slice_duration = float(audio_slice.shape[0] / sr)
   605→            bucket = duration_bucket(slice_duration)
   606→            if bucket is None:
   607→                if slice_duration < 3.0:
   608→                    stats["dropped_too_short"] += 1
   609→                else:
   610→                    stats["dropped_too_long"] += 1
   611→                return
   612→            if metrics["clipping_fraction"] > 0.1:
   613→                stats["dropped_clipped"] += 1
   614→                return
   615→
   616→            duration_bucket_name, base_weight = bucket
   617→            sampling_weight = base_weight * (0.5 if has_music else 1.0)
   618→            quality_score = compute_quality_score(metrics, has_music)
   619→            effective_nyquist = metrics["effective_nyquist_hz"]
   620→
   621→            base_metadata = {
   622→                "id": sample_id,
   623→                "video_id": video_id,
   624→                "language": language,
   625→                "segment_index": segment_index,
   626→                "chunk_index": chunk_index,
   627→                "start": start,
   628→                "end": end,
   629→                "original_start": original_start,
   630→                "original_end": original_end,
   631→                "duration_sec": slice_duration,
   632→                "native_sr": sr,
   633→                "effective_nyquist_hz": effective_nyquist,
   634→                "spectral_rolloff_hz": metrics["spectral_rolloff_hz"],
   635→                "high_freq_energy_ratio": metrics["high_freq_energy_ratio"],
   636→                "is_truly_wideband": bool(effective_nyquist >= 12000),
   637→                "native_sr_bucket": classify_native_bucket(effective_nyquist),
   638→                "loudness_lufs": metrics["loudness_lufs"],
   639→                "loudness_rms_db": metrics["loudness_rms_db"],
   640→                "clipping_fraction": metrics["clipping_fraction"],
   641→                "snr_proxy": metrics["snr_proxy"],
   642→                "quality_score": quality_score,
   643→                "music_prob": float(music_stats.get("music_mean", 0.0)),
   644→                "music_stats": music_stats,
   645→                "speaker": speaker,
   646→                # VAD regions: Full duration placeholder; random 6s crops happen during training.
   647→                "vad_speech_regions": [[0.0, slice_duration]],
   648→                "duration_bucket": duration_bucket_name,
   649→                "sampling_weight": sampling_weight,
   650→            }
   651→
   652→            for target_sr in target_srs:
   653→                resampled = resample_audio(audio_slice, sr, target_sr)
   654→                resampled = np.clip(resampled, -1.0, 1.0)
   655→                audio_bytes = encode_audio(resampled, target_sr, output_format)
   656→                meta = dict(base_metadata)
   657→                meta["sample_rate"] = target_sr
   658→                meta["resample_target_sr"] = target_sr
   659→                outputs.append(SegmentOutput(target_sr, audio_bytes, meta))
   660→
   661→            if target_srs == [16000]:
   662→                stats["chunks_emitted"] += 1
   663→
   664→        sample_id_base = f"{video_id}__{speaker}_{index:04d}"
   665→        use_chunk_suffix = len(windows) > 1
   666→        if use_chunk_suffix:
   667→            stats["segments_chunked"] += 1
   668→
   669→        segment_start = segment_meta.get("start")
   670→        segment_end = segment_meta.get("end")
   671→        original_start = segment_meta.get("original_start")
   672→        original_end = segment_meta.get("original_end")
   673→
   674→        # Always emit 16k outputs; for long segments, emit chunked windows to boost sample count.
   675→        for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
   676→            start_idx = int(round(offset_sec * sr))
   677→            end_idx = int(round((offset_sec + chunk_len_sec) * sr))
   678→            audio_slice = audio[start_idx:end_idx]
   679→            if audio_slice.size == 0:
   680→                continue
   681→            slice_duration = float(audio_slice.shape[0] / sr)
   682→            slice_metrics = compute_audio_metrics(audio_slice, sr)
   683→
   684→            chunk_start = float(segment_start) + offset_sec if segment_start is not None else None
   685→            chunk_end = chunk_start + slice_duration if chunk_start is not None else None
   686→            chunk_original_start = (
   687→                float(original_start) + offset_sec if original_start is not None else None
   688→            )
   689→            chunk_original_end = (
   690→                chunk_original_start + slice_duration
   691→                if chunk_original_start is not None
   692→                else None
   693→            )
   694→
   695→            sample_id = sample_id_base
   696→            chunk_index = None
   697→            if use_chunk_suffix:
   698→                sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
   699→                chunk_index = chunk_idx
   700→
   701→            emit_outputs(
   702→                audio_slice,
   703→                slice_metrics,
   704→                sample_id,
   705→                index,
   706→                chunk_index,
   707→                chunk_start,
   708→                chunk_end,
   709→                chunk_original_start,
   710→                chunk_original_end,
   711→                [16000],
   712→            )
   713→
   714→        # Optional high-rate buckets: use full segment unless configured otherwise.
   715→        full_metrics = compute_audio_metrics(audio, sr)
   716→        effective_nyquist = full_metrics["effective_nyquist_hz"]
   717→        targets_high: list[int] = []
   718→        if effective_nyquist >= 8000:
   719→            targets_high.append(24000)
   720→        if effective_nyquist >= 12000:
   721→            targets_high.append(32000)
   722→        if effective_nyquist >= 16000:
   723→            targets_high.append(44100)
   724→
   725→        if targets_high:
   726→            if chunk_cfg.all_buckets and use_chunk_suffix:
   727→                for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
   728→                    start_idx = int(round(offset_sec * sr))
   729→                    end_idx = int(round((offset_sec + chunk_len_sec) * sr))
   730→                    audio_slice = audio[start_idx:end_idx]
   731→                    if audio_slice.size == 0:
   732→                        continue
   733→                    slice_duration = float(audio_slice.shape[0] / sr)
   734→                    slice_metrics = compute_audio_metrics(audio_slice, sr)
   735→                    slice_effective_nyquist = slice_metrics["effective_nyquist_hz"]
   736→                    slice_targets: list[int] = []
   737→                    if slice_effective_nyquist >= 8000:
   738→                        slice_targets.append(24000)
   739→                    if slice_effective_nyquist >= 12000:
   740→                        slice_targets.append(32000)
   741→                    if slice_effective_nyquist >= 16000:
   742→                        slice_targets.append(44100)
   743→                    if not slice_targets:
   744→                        continue
   745→
   746→                    chunk_start = (
   747→                        float(segment_start) + offset_sec if segment_start is not None else None
   748→                    )
   749→                    chunk_end = (
   750→                        chunk_start + slice_duration if chunk_start is not None else None
   751→                    )
   752→                    chunk_original_start = (
   753→                        float(original_start) + offset_sec if original_start is not None else None
   754→                    )
   755→                    chunk_original_end = (
   756→                        chunk_original_start + slice_duration
   757→                        if chunk_original_start is not None
   758→                        else None
   759→                    )
   760→
   761→                    sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
   762→                    emit_outputs(
   763→                        audio_slice,
   764→                        slice_metrics,
   765→                        sample_id,
   766→                        index,
   767→                        chunk_idx,
   768→                        chunk_start,
   769→                        chunk_end,
   770→                        chunk_original_start,
   771→                        chunk_original_end,
   772→                        slice_targets,
   773→                    )
   774→            else:
   775→                emit_outputs(
   776→                    audio,
   777→                    full_metrics,
   778→                    sample_id_base,
   779→                    index,
   780→                    None,
   781→                    segment_start,
   782→                    segment_end,
   783→                    original_start,
   784→                    original_end,
   785→                    targets_high,
   786→                )
   787→
   788→        return outputs, stats
   789→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

def process_segment(
    video_id: str,
    language: str,
    segment_bytes: bytes,
    segment_meta: dict[str, Any],
    speaker: str,
    index: int,
    chunk_cfg: ChunkConfig,
    silence_cfg: SilenceConfig,
    rng_seed: int,
    output_format: str,
) -> tuple[list[SegmentOutput], dict[str, int]]:
    with timer("segment_total"):
        stats = {
            "dropped_too_short": 0,
            "dropped_too_long": 0,
            "dropped_clipped": 0,
            "chunks_emitted": 0,
            "segments_chunked": 0,
        }
        outputs: list[SegmentOutput] = []

        with timer("flac_decode"):
            try:
                audio, sr = sf.read(io.BytesIO(segment_bytes), dtype="float32")
            except Exception:
                return outputs, stats

        if audio.ndim > 1:
            audio = np.mean(audio, axis=1)

        rng = random.Random(rng_seed)
        windows = make_chunk_windows(audio, sr, chunk_cfg, silence_cfg, rng)

        music_stats = segment_meta.get("music_stats") or {}
        has_music = bool(music_stats.get("has_music", False))

        def emit_outputs(
            audio_slice: np.ndarray,
            metrics: dict[str, float],
            sample_id: str,
            segment_index: int,
            chunk_index: Optional[int],
            start: Optional[float],
            end: Optional[float],
            original_start: Optional[float],
            original_end: Optional[float],
            target_srs: list[int],
        ) -> None:
            slice_duration = float(audio_slice.shape[0] / sr)
            bucket = duration_bucket(slice_duration)
            if bucket is None:
                if slice_duration < 3.0:
                    stats["dropped_too_short"] += 1
                else:
                    stats["dropped_too_long"] += 1
                return
            if metrics["clipping_fraction"] > 0.1:
                stats["dropped_clipped"] += 1
                return

            duration_bucket_name, base_weight = bucket
            sampling_weight = base_weight * (0.5 if has_music else 1.0)
            quality_score = compute_quality_score(metrics, has_music)
            effective_nyquist = metrics["effective_nyquist_hz"]

            base_metadata = {
                "id": sample_id,
                "video_id": video_id,
                "language": language,
                "segment_index": segment_index,
                "chunk_index": chunk_index,
                "start": start,
                "end": end,
                "original_start": original_start,
                "original_end": original_end,
                "duration_sec": slice_duration,
                "native_sr": sr,
                "effective_nyquist_hz": effective_nyquist,
                "spectral_rolloff_hz": metrics["spectral_rolloff_hz"],
                "high_freq_energy_ratio": metrics["high_freq_energy_ratio"],
                "is_truly_wideband": bool(effective_nyquist >= 12000),
                "native_sr_bucket": classify_native_bucket(effective_nyquist),
                "loudness_lufs": metrics["loudness_lufs"],
                "loudness_rms_db": metrics["loudness_rms_db"],
                "clipping_fraction": metrics["clipping_fraction"],
                "snr_proxy": metrics["snr_proxy"],
                "quality_score": quality_score,
                "music_prob": float(music_stats.get("music_mean", 0.0)),
                "music_stats": music_stats,
                "speaker": speaker,
                # VAD regions: Full duration placeholder; random 6s crops happen during training.
                "vad_speech_regions": [[0.0, slice_duration]],
                "duration_bucket": duration_bucket_name,
                "sampling_weight": sampling_weight,
            }

            for target_sr in target_srs:
                resampled = resample_audio(audio_slice, sr, target_sr)
                resampled = np.clip(resampled, -1.0, 1.0)
                audio_bytes = encode_audio(resampled, target_sr, output_format)
                meta = dict(base_metadata)
                meta["sample_rate"] = target_sr
                meta["resample_target_sr"] = target_sr
                outputs.append(SegmentOutput(target_sr, audio_bytes, meta))

            if target_srs == [16000]:
                stats["chunks_emitted"] += 1

        sample_id_base = f"{video_id}__{speaker}_{index:04d}"
        use_chunk_suffix = len(windows) > 1
        if use_chunk_suffix:
            stats["segments_chunked"] += 1

        segment_start = segment_meta.get("start")
        segment_end = segment_meta.get("end")
        original_start = segment_meta.get("original_start")
        original_end = segment_meta.get("original_end")

        # Always emit 16k outputs; for long segments, emit chunked windows to boost sample count.
        for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
            start_idx = int(round(offset_sec * sr))
            end_idx = int(round((offset_sec + chunk_len_sec) * sr))
            audio_slice = audio[start_idx:end_idx]
            if audio_slice.size == 0:
                continue
            slice_duration = float(audio_slice.shape[0] / sr)
            slice_metrics = compute_audio_metrics(audio_slice, sr)

            chunk_start = float(segment_start) + offset_sec if segment_start is not None else None
            chunk_end = chunk_start + slice_duration if chunk_start is not None else None
            chunk_original_start = (
                float(original_start) + offset_sec if original_start is not None else None
            )
            chunk_original_end = (
                chunk_original_start + slice_duration
                if chunk_original_start is not None
                else None
            )

            sample_id = sample_id_base
            chunk_index = None
            if use_chunk_suffix:
                sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
                chunk_index = chunk_idx

            emit_outputs(
                audio_slice,
                slice_metrics,
                sample_id,
                index,
                chunk_index,
                chunk_start,
                chunk_end,
                chunk_original_start,
                chunk_original_end,
                [16000],
            )

        # Optional high-rate buckets: use full segment unless configured otherwise.
        full_metrics = compute_audio_metrics(audio, sr)
        effective_nyquist = full_metrics["effective_nyquist_hz"]
        targets_high: list[int] = []
        if effective_nyquist >= 8000:
            targets_high.append(24000)
        if effective_nyquist >= 12000:
            targets_high.append(32000)
        if effective_nyquist >= 16000:
            targets_high.append(44100)

        if targets_high:
            if chunk_cfg.all_buckets and use_chunk_suffix:
                for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
                    start_idx = int(round(offset_sec * sr))
                    end_idx = int(round((offset_sec + chunk_len_sec) * sr))
                    audio_slice = audio[start_idx:end_idx]
                    if audio_slice.size == 0:
                        continue
                    slice_duration = float(audio_slice.shape[0] / sr)
                    slice_metrics = compute_audio_metrics(audio_slice, sr)
                    slice_effective_nyquist = slice_metrics["effective_nyquist_hz"]
                    slice_targets: list[int] = []
                    if slice_effective_nyquist >= 8000:
                        slice_targets.append(24000)
                    if slice_effective_nyquist >= 12000:
                        slice_targets.append(32000)
                    if slice_effective_nyquist >= 16000:
                        slice_targets.append(44100)
                    if not slice_targets:
                        continue

                    chunk_start = (
                        float(segment_start) + offset_sec if segment_start is not None else None
                    )
                    chunk_end = (
                        chunk_start + slice_duration if chunk_start is not None else None
                    )
                    chunk_original_start = (
                        float(original_start) + offset_sec if original_start is not None else None
                    )
                    chunk_original_end = (
                        chunk_original_start + slice_duration
                        if chunk_original_start is not None
                        else None
                    )

                    sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
                    emit_outputs(
                        audio_slice,
                        slice_metrics,
                        sample_id,
                        index,
                        chunk_idx,
                        chunk_start,
                        chunk_end,
                        chunk_original_start,
                        chunk_original_end,
                        slice_targets,
                    )
            else:
                emit_outputs(
                    audio,
                    full_metrics,
                    sample_id_base,
                    index,
                    None,
                    segment_start,
                    segment_end,
                    original_start,
                    original_end,
                    targets_high,
                )

        return outputs, stats


Now let me examine the worker architecture and main processing loop:


  1240→
  1241→    segment_workers = max(1, args.segment_workers)
  1242→    max_inflight = (
  1243→        args.segment_max_inflight
  1244→        if args.segment_max_inflight > 0
  1245→        else max(128, segment_workers * 2)
  1246→    )
  1247→    pending: set[Any] = set()
  1248→    future_info: dict[Any, bool] = {}
  1249→
  1250→    def handle_future(future) -> None:
  1251→        is_val = future_info.pop(future, False)
  1252→        outputs, seg_stats = future.result()
  1253→        stats["dropped_too_short"] += seg_stats["dropped_too_short"]
  1254→        stats["dropped_too_long"] += seg_stats["dropped_too_long"]
  1255→        stats["dropped_clipped"] += seg_stats["dropped_clipped"]
  1256→        stats["segments_chunked"] += seg_stats.get("segments_chunked", 0)
  1257→        stats["chunks_emitted"] += seg_stats.get("chunks_emitted", 0)
  1258→        for out in outputs:
  1259→            stats["segments_kept"][out.sample_rate] += 1
  1260→            if out.sample_rate == 16000:
  1261→                bucket = out.metadata.get("native_sr_bucket")
  1262→                if bucket:
  1263→                    stats["native_sr_buckets"][bucket] = (
  1264→                        stats["native_sr_buckets"].get(bucket, 0) + 1
  1265→                    )
  1266→            writer = writers[out.sample_rate]
  1267→            with timer("shard_write"):
  1268→                shard_name = writer.add_sample(
  1269→                    out.metadata["id"], out.audio_bytes, out.metadata
  1270→                )
  1271→            manifest_entry = dict(out.metadata)
  1272→            manifest_entry["shard"] = f"{shard_name}.tar"
  1273→            manifest_entry["audio"] = f"{out.metadata['id']}.{audio_ext}"
  1274→            line = json.dumps(manifest_entry) + "\n"
  1275→            if is_val:
  1276→                val_manifest.write(line)
  1277→            else:
  1278→                train_manifest.write(line)
  1279→
  1280→    def drain_completed() -> None:
  1281→        nonlocal pending
  1282→        if not pending:
  1283→            return
  1284→        done, pending = wait(pending, return_when=FIRST_COMPLETED)
  1285→        for future in done:
  1286→            handle_future(future)
  1287→
  1288→    with ThreadPoolExecutor(max_workers=segment_workers) as executor:
  1289→        for video_id, tar_path in tqdm(download_paths.items(), desc="process"):
  1290→            stats["videos"] += 1
  1291→            is_val = video_id in val_ids
  1292→            with timer("tar_open"):
  1293→                tf = tarfile.open(tar_path, "r")
  1294→            with tf:
  1295→                with timer("tar_meta_load"):
  1296→                    meta = json.loads(
  1297→                        tf.extractfile(f"{video_id}/metadata.json").read()
  1298→                    )
  1299→                usable_segments = [
  1300→                    s for s in meta.get("segments", []) if s.get("status") == "usable"
  1301→                ]
  1302→                # Create lookup by (speaker, start) for robust matching
  1303→                # The filename contains 'start' timestamp (relative to intro-skipped audio),
  1304→                # not 'original_start' (absolute time in original video)
  1305→                with timer("tar_build_segment_map"):
  1306→                    segment_map = {
  1307→                        (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
  1308→                        for seg in usable_segments
  1309→                    }
  1310→
  1311→                with timer("tar_list_members"):
  1312→                    members = [m for m in tf.getmembers() if m.name.endswith(".flac")]
  1313→                stats["segments_total"] += len(members)
  1314→
  1315→                for member in members:
  1316→                    parsed = parse_segment_filename(member.name)
  1317→                    if not parsed:
  1318→                        continue
  1319→                    speaker, start_time = parsed
  1320→                    # Match by (speaker, start) with rounding for float precision
  1321→                    seg_meta = segment_map.get((speaker, round(start_time, 2)))
  1322→                    if not seg_meta:
  1323→                        # Try fuzzy match for edge cases (tolerance 0.1s)
  1324→                        matches = [
  1325→                            s for s in usable_segments
  1326→                            if s.get("speaker") == speaker
  1327→                            and abs(s.get("start", 0.0) - start_time) < 0.1
  1328→                        ]
  1329→                        if not matches:
  1330→                            continue
  1331→                        seg_meta = matches[0]
  1332→
  1333→                    while len(pending) >= max_inflight:
  1334→                        drain_completed()
  1335→
  1336→                    with timer("tar_extract_segment"):
  1337→                        segment_bytes = tf.extractfile(member).read()
  1338→                    # Extract index from filename for ID generation (cosmetic only)
  1339→                    filename_parts = member.name.split("_")
  1340→                    try:
  1341→                        index = int(filename_parts[-2])
  1342→                    except (ValueError, IndexError):
  1343→                        index = 0
  1344→                    rng_seed = make_segment_seed(args.seed, video_id, speaker, index)
  1345→
  1346→                    future = executor.submit(
  1347→                        process_segment,
  1348→                        video_id,
  1349→                        language,
  1350→                        segment_bytes,
  1351→                        seg_meta,
  1352→                        speaker,
  1353→                        index,
  1354→                        chunk_cfg,
  1355→                        silence_cfg,
  1356→                        rng_seed,
  1357→                        output_format,
  1358→                    )
  1359→                    future_info[future] = is_val
  1360→                    pending.add(future)
  1361→
  1362→        while pending:
  1363→            drain_completed()
  1364→
  1365→    for writer in writers.values():
  1366→        writer.close()
  1367→    train_manifest.close()
  1368→    val_manifest.close()
  1369→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

    segment_workers = max(1, args.segment_workers)
    max_inflight = (
        args.segment_max_inflight
        if args.segment_max_inflight > 0
        else max(128, segment_workers * 2)
    )
    pending: set[Any] = set()
    future_info: dict[Any, bool] = {}

    def handle_future(future) -> None:
        is_val = future_info.pop(future, False)
        outputs, seg_stats = future.result()
        stats["dropped_too_short"] += seg_stats["dropped_too_short"]
        stats["dropped_too_long"] += seg_stats["dropped_too_long"]
        stats["dropped_clipped"] += seg_stats["dropped_clipped"]
        stats["segments_chunked"] += seg_stats.get("segments_chunked", 0)
        stats["chunks_emitted"] += seg_stats.get("chunks_emitted", 0)
        for out in outputs:
            stats["segments_kept"][out.sample_rate] += 1
            if out.sample_rate == 16000:
                bucket = out.metadata.get("native_sr_bucket")
                if bucket:
                    stats["native_sr_buckets"][bucket] = (
                        stats["native_sr_buckets"].get(bucket, 0) + 1
                    )
            writer = writers[out.sample_rate]
            with timer("shard_write"):
                shard_name = writer.add_sample(
                    out.metadata["id"], out.audio_bytes, out.metadata
                )
            manifest_entry = dict(out.metadata)
            manifest_entry["shard"] = f"{shard_name}.tar"
            manifest_entry["audio"] = f"{out.metadata['id']}.{audio_ext}"
            line = json.dumps(manifest_entry) + "\n"
            if is_val:
                val_manifest.write(line)
            else:
                train_manifest.write(line)

    def drain_completed() -> None:
        nonlocal pending
        if not pending:
            return
        done, pending = wait(pending, return_when=FIRST_COMPLETED)
        for future in done:
            handle_future(future)

    with ThreadPoolExecutor(max_workers=segment_workers) as executor:
        for video_id, tar_path in tqdm(download_paths.items(), desc="process"):
            stats["videos"] += 1
            is_val = video_id in val_ids
            with timer("tar_open"):
                tf = tarfile.open(tar_path, "r")
            with tf:
                with timer("tar_meta_load"):
                    meta = json.loads(
                        tf.extractfile(f"{video_id}/metadata.json").read()
                    )
                usable_segments = [
                    s for s in meta.get("segments", []) if s.get("status") == "usable"
                ]
                # Create lookup by (speaker, start) for robust matching
                # The filename contains 'start' timestamp (relative to intro-skipped audio),
                # not 'original_start' (absolute time in original video)
                with timer("tar_build_segment_map"):
                    segment_map = {
                        (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
                        for seg in usable_segments
                    }

                with timer("tar_list_members"):
                    members = [m for m in tf.getmembers() if m.name.endswith(".flac")]
                stats["segments_total"] += len(members)

                for member in members:
                    parsed = parse_segment_filename(member.name)
                    if not parsed:
                        continue
                    speaker, start_time = parsed
                    # Match by (speaker, start) with rounding for float precision
                    seg_meta = segment_map.get((speaker, round(start_time, 2)))
                    if not seg_meta:
                        # Try fuzzy match for edge cases (tolerance 0.1s)
                        matches = [
                            s for s in usable_segments
                            if s.get("speaker") == speaker
                            and abs(s.get("start", 0.0) - start_time) < 0.1
                        ]
                        if not matches:
                            continue
                        seg_meta = matches[0]

                    while len(pending) >= max_inflight:
                        drain_completed()

                    with timer("tar_extract_segment"):
                        segment_bytes = tf.extractfile(member).read()
                    # Extract index from filename for ID generation (cosmetic only)
                    filename_parts = member.name.split("_")
                    try:
                        index = int(filename_parts[-2])
                    except (ValueError, IndexError):
                        index = 0
                    rng_seed = make_segment_seed(args.seed, video_id, speaker, index)

                    future = executor.submit(
                        process_segment,
                        video_id,
                        language,
                        segment_bytes,
                        seg_meta,
                        speaker,
                        index,
                        chunk_cfg,
                        silence_cfg,
                        rng_seed,
                        output_format,
                    )
                    future_info[future] = is_val
                    pending.add(future)

        while pending:
            drain_completed()

    for writer in writers.values():
        writer.close()
    train_manifest.close()
    val_manifest.close()


   390→
   391→def compute_frame_rms(
   392→    audio: np.ndarray, sr: int, frame_sec: float, hop_sec: float
   393→) -> tuple[np.ndarray, np.ndarray]:
   394→    frame_len = max(1, int(round(frame_sec * sr)))
   395→    hop_len = max(1, int(round(hop_sec * sr)))
   396→    if audio.size < frame_len:
   397→        return np.array([], dtype=np.float32), np.array([], dtype=np.float32)
   398→
   399→    if not audio.flags["C_CONTIGUOUS"]:
   400→        audio = np.ascontiguousarray(audio)
   401→
   402→    num_frames = 1 + (audio.size - frame_len) // hop_len
   403→    if num_frames <= 0:
   404→        return np.array([], dtype=np.float32), np.array([], dtype=np.float32)
   405→
   406→    stride = audio.strides[0]
   407→    frames = np.lib.stride_tricks.as_strided(
   408→        audio,
   409→        shape=(num_frames, frame_len),
   410→        strides=(hop_len * stride, stride),
   411→    )
   412→    rms = np.sqrt(np.mean(frames * frames, axis=1) + 1e-12)
   413→    times = (np.arange(num_frames, dtype=np.float32) * hop_len + (frame_len / 2)) / sr
   414→    return times.astype(np.float32), rms.astype(np.float32)
   415→
   416→
   417→def find_silence_boundary(
   418→    times: np.ndarray,
   419→    rms: np.ndarray,
   420→    search_start: float,
   421→    search_end: float,
   422→    threshold_rms: float,
   423→    median_db: float,
   424→    db_drop: float,
   425→) -> Optional[float]:
   426→    if times.size == 0:
   427→        return None
   428→    mask = (times >= search_start) & (times <= search_end)
   429→    if not np.any(mask):
   430→        return None
   431→    subset = rms[mask]
   432→    min_idx = int(np.argmin(subset))
   433→    min_rms = float(subset[min_idx])
   434→    min_time = float(times[mask][min_idx])
   435→    min_db = 20.0 * math.log10(min_rms + 1e-12)
   436→    if min_rms <= threshold_rms or (median_db - min_db) >= db_drop:
   437→        return min_time
   438→    return None
   439→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

def compute_frame_rms(
    audio: np.ndarray, sr: int, frame_sec: float, hop_sec: float
) -> tuple[np.ndarray, np.ndarray]:
    frame_len = max(1, int(round(frame_sec * sr)))
    hop_len = max(1, int(round(hop_sec * sr)))
    if audio.size < frame_len:
        return np.array([], dtype=np.float32), np.array([], dtype=np.float32)

    if not audio.flags["C_CONTIGUOUS"]:
        audio = np.ascontiguousarray(audio)

    num_frames = 1 + (audio.size - frame_len) // hop_len
    if num_frames <= 0:
        return np.array([], dtype=np.float32), np.array([], dtype=np.float32)

    stride = audio.strides[0]
    frames = np.lib.stride_tricks.as_strided(
        audio,
        shape=(num_frames, frame_len),
        strides=(hop_len * stride, stride),
    )
    rms = np.sqrt(np.mean(frames * frames, axis=1) + 1e-12)
    times = (np.arange(num_frames, dtype=np.float32) * hop_len + (frame_len / 2)) / sr
    return times.astype(np.float32), rms.astype(np.float32)

def find_silence_boundary(
    times: np.ndarray,
    rms: np.ndarray,
    search_start: float,
    search_end: float,
    threshold_rms: float,
    median_db: float,
    db_drop: float,
) -> Optional[float]:
    if times.size == 0:
        return None
    mask = (times >= search_start) & (times <= search_end)
    if not np.any(mask):
        return None
    subset = rms[mask]
    min_idx = int(np.argmin(subset))
    min_rms = float(subset[min_idx])
    min_time = float(times[mask][min_idx])
    min_db = 20.0 * math.log10(min_rms + 1e-12)
    if min_rms <= threshold_rms or (median_db - min_db) >= db_drop:
        return min_time
    return None


Perfect! Now let me check the ShardWriter and look for any additional implementation details:


   791→class ShardWriter:
   792→    def __init__(
   793→        self, output_dir: Path, prefix: str, target_size_bytes: int, audio_ext: str
   794→    ) -> None:
   795→        self.output_dir = output_dir
   796→        self.prefix = prefix
   797→        self.target_size_bytes = target_size_bytes
   798→        self.audio_ext = audio_ext
   799→        self.shard_index = 0
   800→        self.current_tar: Optional[tarfile.TarFile] = None
   801→        self.current_jsonl = None
   802→        self.current_size = 0
   803→        self.current_count = 0
   804→        self.current_name = ""
   805→        self.output_dir.mkdir(parents=True, exist_ok=True)
   806→
   807→    def _open_new_shard(self) -> None:
   808→        if self.current_tar:
   809→            self.close()
   810→        shard_name = f"{self.prefix}-{self.shard_index:06d}"
   811→        tar_path = self.output_dir / f"{shard_name}.tar"
   812→        jsonl_path = self.output_dir / f"{shard_name}.jsonl"
   813→        self.current_tar = tarfile.open(tar_path, "w")
   814→        self.current_jsonl = jsonl_path.open("w", encoding="utf-8")
   815→        self.current_size = 0
   816→        self.current_count = 0
   817→        self.current_name = shard_name
   818→        self.shard_index += 1
   819→
   820→    def add_sample(self, sample_id: str, audio_bytes: bytes, metadata: dict[str, Any]) -> str:
   821→        if self.current_tar is None:
   822→            self._open_new_shard()
   823→        if (
   824→            self.current_count > 0
   825→            and self.current_size + len(audio_bytes) > self.target_size_bytes
   826→        ):
   827→            self._open_new_shard()
   828→
   829→        assert self.current_tar is not None
   830→        assert self.current_jsonl is not None
   831→
   832→        arcname = f"{sample_id}.{self.audio_ext}"
   833→        info = tarfile.TarInfo(name=arcname)
   834→        info.size = len(audio_bytes)
   835→        info.mtime = int(time.time())
   836→        self.current_tar.addfile(info, io.BytesIO(audio_bytes))
   837→        self.current_jsonl.write(json.dumps(metadata) + "\n")
   838→
   839→        self.current_size += len(audio_bytes)
   840→        self.current_count += 1
   841→        return self.current_name
   842→
   843→    def close(self) -> None:
   844→        if self.current_tar:
   845→            self.current_tar.close()
   846→        if self.current_jsonl:
   847→            self.current_jsonl.close()
   848→        self.current_tar = None
   849→        self.current_jsonl = None
   850→
   851→
   852→def create_prefix_placeholders(
   853→    s3,
   854→    bucket: str,
   855→    languages: list[str],

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

class ShardWriter:
    def __init__(
        self, output_dir: Path, prefix: str, target_size_bytes: int, audio_ext: str
    ) -> None:
        self.output_dir = output_dir
        self.prefix = prefix
        self.target_size_bytes = target_size_bytes
        self.audio_ext = audio_ext
        self.shard_index = 0
        self.current_tar: Optional[tarfile.TarFile] = None
        self.current_jsonl = None
        self.current_size = 0
        self.current_count = 0
        self.current_name = ""
        self.output_dir.mkdir(parents=True, exist_ok=True)

    def _open_new_shard(self) -> None:
        if self.current_tar:
            self.close()
        shard_name = f"{self.prefix}-{self.shard_index:06d}"
        tar_path = self.output_dir / f"{shard_name}.tar"
        jsonl_path = self.output_dir / f"{shard_name}.jsonl"
        self.current_tar = tarfile.open(tar_path, "w")
        self.current_jsonl = jsonl_path.open("w", encoding="utf-8")
        self.current_size = 0
        self.current_count = 0
        self.current_name = shard_name
        self.shard_index += 1

    def add_sample(self, sample_id: str, audio_bytes: bytes, metadata: dict[str, Any]) -> str:
        if self.current_tar is None:
            self._open_new_shard()
        if (
            self.current_count > 0
            and self.current_size + len(audio_bytes) > self.target_size_bytes
        ):
            self._open_new_shard()

        assert self.current_tar is not None
        assert self.current_jsonl is not None

        arcname = f"{sample_id}.{self.audio_ext}"
        info = tarfile.TarInfo(name=arcname)
        info.size = len(audio_bytes)
        info.mtime = int(time.time())
        self.current_tar.addfile(info, io.BytesIO(audio_bytes))
        self.current_jsonl.write(json.dumps(metadata) + "\n")

        self.current_size += len(audio_bytes)
        self.current_count += 1
        return self.current_name

    def close(self) -> None:
        if self.current_tar:
            self.current_tar.close()
        if self.current_jsonl:
            self.current_jsonl.close()
        self.current_tar = None
        self.current_jsonl = None

def create_prefix_placeholders(
    s3,
    bucket: str,
    languages: list[str],


Excellent! Now I have a comprehensive understanding of the pipeline. Let me create a detailed analysis report:

## Audio Processing Pipeline Analysis - prepare_shards.py

Based on my thorough exploration of the codebase, here's a comprehensive breakdown of the audio processing pipeline:

---

### 1. FLAC DECODING

**Implementation Location:** Lines 577-584 in `process_segment()`

```python
with timer("flac_decode"):
    try:
        audio, sr = sf.read(io.BytesIO(segment_bytes), dtype="float32")
    except Exception:
        return outputs, stats

if audio.ndim > 1:
    audio = np.mean(audio, axis=1)
```

**Details:**
- **Library:** `soundfile` (sf)
- **Method:** `sf.read()` with in-memory BytesIO buffer
- **Format:** FLAC compressed audio files
- **Output:**
  - Float32 numpy arrays
  - Mono conversion via mean() for multi-channel audio
  - Sample rate preserved from file
- **Error Handling:** Silently fails with empty outputs

---

### 2. LUFS COMPUTATION (Loudness Units relative to Full Scale)

**Implementation Location:** Lines 318-326 in `compute_audio_metrics()`

```python
with timer("audio_metrics_lufs"):
    loudness_lufs = -100.0
    if pyln is not None:
        try:
            meter = get_lufs_meter(sr)
            if meter is not None:
                loudness_lufs = float(meter.integrated_loudness(audio))
        except Exception:
            loudness_lufs = -100.0
```

**Supporting Function:** Lines 117-128

```python
def get_lufs_meter(sr: int) -> Optional[Any]:
    if pyln is None:
        return None
    meters = getattr(_lufs_local, "meters", None)
    if meters is None:
        meters = {}
        _lufs_local.meters = meters
    meter = meters.get(sr)
    if meter is None:
        meter = pyln.Meter(sr)
        meters[sr] = meter
    return meter
```

**Details:**
- **Library:** `pyloudnorm` (pyln) - optional dependency
- **Caching:** Thread-local storage with per-sample-rate meter instances
- **Calculation:** ITU-R BS.1770 integrated loudness standard
- **Fallback:** Returns -100.0 dB if unavailable
- **Thread Safety:** Uses `threading.local()` for thread-safe meter instances

---

### 3. SPECTRAL COMPUTATION

**Implementation Location:** Lines 266-288 in `compute_spectral_metrics()`

```python
def compute_spectral_metrics(audio: np.ndarray, sr: int) -> tuple[float, float, float]:
    if audio.size == 0:
        return 0.0, 0.0, 0.0
    with timer("cpu_spectral_compute"):
        max_samples = min(audio.size, sr * 5)
        window = get_hann_window(max_samples)
        chunk = audio[:max_samples] * window
        spec = np.fft.rfft(chunk)
        power = np.abs(spec) ** 2
        freqs = get_rfftfreq(max_samples, sr)
        total_energy = float(np.sum(power)) + 1e-12
        cumulative = np.cumsum(power)
        rolloff_idx = int(np.searchsorted(cumulative, 0.85 * cumulative[-1]))
        rolloff_hz = float(freqs[min(rolloff_idx, len(freqs) - 1)])

        power_db = 10.0 * np.log10(power + 1e-12)
        threshold = power_db.max() - 60.0
        above = np.where(power_db >= threshold)[0]
        effective_nyquist_hz = float(freqs[above[-1]]) if above.size > 0 else 0.0

        high_freq_energy = float(np.sum(power[freqs >= 8000.0]))
        high_freq_ratio = high_freq_energy / total_energy
    return rolloff_hz, effective_nyquist_hz, high_freq_ratio
```

**Caching Functions:** Lines 92-114

```python
def get_hann_window(length: int) -> np.ndarray:
    # Dict-based cache with lock
    window = _window_cache.get(length)
    if window is not None:
        return window
    with _cache_lock:
        window = _window_cache.get(length)
        if window is None:
            window = np.hanning(length)
            _window_cache[length] = window
    return window

def get_rfftfreq(length: int, sr: int) -> np.ndarray:
    # Similar caching for frequency arrays
    key = (length, sr)
    freqs = _freq_cache.get(key)
    if freqs is not None:
        return freqs
    with _cache_lock:
        freqs = _freq_cache.get(key)
        if freqs is None:
            freqs = np.fft.rfftfreq(length, d=1.0 / sr)
            _freq_cache[key] = freqs
    return freqs
```

**Details:**
- **Window Function:** Hann window applied to first 5 seconds
- **FFT:** Real FFT (rfft) for efficiency
- **Power Spectrum:** Magnitude squared, with 1e-12 floor for numerical stability
- **Metrics Computed:**
  1. **Spectral Rolloff:** Frequency containing 85% of cumulative energy
  2. **Effective Nyquist Frequency:** Highest frequency within 60dB of peak power
  3. **High-Frequency Energy Ratio:** Energy above 8kHz as fraction of total
- **Caching:** Thread-safe caching for windows and frequency arrays using double-check locking
- **Output:** 3-tuple (rolloff_hz, effective_nyquist_hz, high_freq_ratio)

**Additional Metrics in compute_audio_metrics():** Lines 308-339

```python
def compute_audio_metrics(audio: np.ndarray, sr: int) -> dict[str, float]:
    with timer("audio_metrics_basic"):
        audio = audio.astype(np.float32, copy=False)
        rms = float(np.sqrt(np.mean(audio ** 2) + 1e-12))
        loudness_rms_db = 20.0 * math.log10(rms + 1e-12)
        clipping_fraction = float(np.mean(np.abs(audio) > 0.99))
        dc_offset = float(np.mean(audio))
        noise_floor = float(np.percentile(np.abs(audio), 10))
        snr_proxy = 20.0 * math.log10((rms + 1e-9) / (noise_floor + 1e-9))
    # ... LUFS computation ...
    rolloff_hz, effective_nyquist_hz, high_freq_ratio = compute_spectral_metrics(audio, sr)

    return {
        "loudness_lufs": loudness_lufs,
        "loudness_rms_db": loudness_rms_db,
        "clipping_fraction": clipping_fraction,
        "dc_offset": dc_offset,
        "snr_proxy": snr_proxy,
        "spectral_rolloff_hz": rolloff_hz,
        "effective_nyquist_hz": effective_nyquist_hz,
        "high_freq_energy_ratio": high_freq_ratio,
    }
```

---

### 4. AUDIO ENCODING (WAV/FLAC Output)

**Implementation Location:** Lines 367-374

```python
def encode_audio(audio: np.ndarray, sr: int, output_format: str) -> bytes:
    with timer("audio_encode"):
        buf = io.BytesIO()
        if output_format == "wav":
            sf.write(buf, audio, sr, format="WAV", subtype="PCM_16")
        else:
            sf.write(buf, audio, sr, format="FLAC", subtype="PCM_16")
        return buf.getvalue()
```

**Details:**
- **Library:** `soundfile` (sf)
- **Formats:** WAV or FLAC (command-line configurable with `--output-format`)
- **Bit Depth:** PCM_16 (16-bit) for both formats
- **In-Memory Encoding:** Uses BytesIO buffer to return raw bytes
- **Purpose:** Output bytes stored in tar archives for distribution

---

### 5. AUDIO RESAMPLING

**Implementation Location:** Lines 353-364

```python
def resample_audio(
    audio: np.ndarray,
    src_sr: int,
    dst_sr: int,
) -> np.ndarray:
    if src_sr == dst_sr:
        return audio
    with timer("cpu_resample"):
        gcd = math.gcd(src_sr, dst_sr)
        up = dst_sr // gcd
        down = src_sr // gcd
        return signal.resample_poly(audio, up, down).astype(np.float32, copy=False)
```

**Details:**
- **Library:** `scipy.signal.resample_poly` (polyphase resampling)
- **Optimization:** Uses GCD to minimize up/down factors
- **Output Rates:** 16kHz, 24kHz, 32kHz, 44.1kHz
- **Output Type:** Float32

---

### 6. OVERALL WORKER/PROCESSING ARCHITECTURE

**Main Flow:**

1. **Download Phase** (lines 1143-1179):
   - ThreadPoolExecutor with configurable workers (default: 16)
   - Downloads FLAC tar archives from R2 cloud storage
   - Parallel downloads with boto3 TransferConfig

2. **Processing Phase** (lines 1288-1363):
   - ThreadPoolExecutor with segment workers (default: 8)
   - Max in-flight tasks limited to prevent memory overload (default: 128, can be tuned)
   - Task submission loop maintains `max_inflight` limit

3. **Segment Processing Function** `process_segment()` (lines 555-788):
   - Decodes FLAC audio
   - Chunks long audio into 6-10s windows (configurable)
   - Computes metrics for each chunk
   - Emits outputs at multiple sample rates (16k, 24k, 32k, 44.1k)
   - Returns list of SegmentOutput objects

4. **Output Handling** `handle_future()` (lines 1250-1278):
   - Aggregates statistics
   - Writes to ShardWriter for each sample rate
   - Writes to train/val manifests (JSON lines)
   - Thread-safe accumulation of statistics

5. **Shard Writing** `ShardWriter` class (lines 791-849):
   - Maintains separate TAR files per sample rate
   - Tracks shard size, rolls to new shard when threshold exceeded
   - Pairs each TAR with JSONL metadata file
   - Sequential numbering: `shard-{prefix}-{index:06d}.tar`

---

### 7. PARALLELIZATION STRATEGY

**Thread-Level Parallelization:**

```python
with ThreadPoolExecutor(max_workers=segment_workers) as executor:
    for video_id, tar_path in tqdm(download_paths.items(), desc="process"):
        # Main loop: iterates through video TAR files (sequential per-file)
        # For each segment in the TAR:
        while len(pending) >= max_inflight:
            drain_completed()  # Wait for some tasks to complete

        future = executor.submit(
            process_segment,  # CPU-intensive audio processing
            video_id,
            language,
            segment_bytes,
            seg_meta,
            speaker,
            index,
            chunk_cfg,
            silence_cfg,
            rng_seed,
            output_format,
        )
        pending.add(future)

    # Drain remaining tasks
    while pending:
        drain_completed()
```

**Key Features:**
- **Back-pressure Control:** `max_inflight` prevents memory exhaustion
- **Dynamic Waiting:** `wait(pending, return_when=FIRST_COMPLETED)` processes one completed task at a time
- **Two-Stage Pipeline:**
  1. Download stage: Parallel downloads to disk
  2. Processing stage: Parallel segment processing
- **Per-Sample-Rate Writers:** Each sample rate has its own ShardWriter with independent TAR files

**In-Memory Caching:**
- Window cache: `_window_cache` (dict)
- Frequency cache: `_freq_cache` (dict)
- LUFS meters: `_lufs_local` (threading.local)
- All caches protected by `_cache_lock` (threading.Lock)

---

### 8. TIMING/PROFILING CODE

**Profiling Infrastructure** (lines 49-83):

```python
_timing_lock = threading.Lock()
_timing_stats: dict[str, list[float]] = defaultdict(list)
_timing_enabled = False

@contextmanager
def timer(name: str):
    """Context manager for timing code blocks."""
    if not _timing_enabled:
        yield
        return
    start = time.perf_counter()
    try:
        yield
    finally:
        elapsed = time.perf_counter() - start
        with _timing_lock:
            _timing_stats[name].append(elapsed)

def print_timing_stats():
    """Print aggregated timing statistics."""
    if not _timing_stats:
        return
    print("\n=== Timing Statistics ===")
    total_time = sum(sum(times) for times in _timing_stats.values())
    for name, times in sorted(_timing_stats.items()):
        count = len(times)
        total = sum(times)
        avg = total / count if count > 0 else 0
        percent = (total / total_time * 100) if total_time > 0 else 0
        print(f"{name:35s}: {count:6d} calls, {total:8.2f}s total, {avg*1000:8.2f}ms avg, {percent:5.1f}%")
    print(f"{'TOTAL':35s}:              {total_time:8.2f}s")
    print("=" * 80)
```

**Instrumented Code Sections:**
- `timer("segment_total")` - End-to-end segment processing
- `timer("flac_decode")` - FLAC decoding time
- `timer("cpu_spectral_compute")` - FFT and spectral metrics
- `timer("audio_metrics_basic")` - RMS, clipping, DC offset, SNR
- `timer("audio_metrics_lufs")` - LUFS computation
- `timer("cpu_resample")` - Audio resampling
- `timer("audio_encode")` - WAV/FLAC encoding
- `timer("shard_write")` - Writing to TAR/JSONL
- `timer("tar_open")`, `timer("tar_extract_segment")`, `timer("tar_meta_load")` - TAR operations
- `timer("r2_download_total")`, `timer("r2_head_object")` - Cloud storage operations

**Activation:** Use `--profile-timing` flag to enable collection and `print_timing_stats()` called at end of main

---

### 9. DATA FLOW SUMMARY

```
Downloaded FLAC TAR
    ↓
[process_segment] (ThreadPoolExecutor)
    ├→ FLAC Decode (soundfile)
    ├→ Convert to mono if needed
    ├→ Create chunk windows (silence detection optional)
    │
    └→ For each chunk/window:
        ├→ Compute audio metrics:
        │   ├→ compute_audio_metrics()
        │   │   ├→ RMS/dB loudness
        │   │   ├→ Clipping detection
        │   │   ├→ DC offset
        │   │   ├→ SNR estimate
        │   │   └→ Spectral metrics (FFT-based)
        │   │       ├→ Rolloff frequency
        │   │       ├→ Effective Nyquist
        │   │       └→ High-freq energy ratio
        │   └→ LUFS computation (pyloudnorm)
        │
        ├→ Quality score computation
        ├→ Emit outputs for each target SR:
        │   ├→ Resample (scipy.signal.resample_poly)
        │   ├→ Clip to [-1.0, 1.0]
        │   └→ Encode (soundfile WAV/FLAC)
        │
        └→ SegmentOutput created for each (SR, encoded_bytes, metadata)
                    ↓
            [handle_future]
                    ↓
            ShardWriter (per-SR tar files)
                    ↓
        Output TAR files + JSONL metadata
```

---

### 10. KEY OPTIMIZATION TECHNIQUES

1. **Windowing Cache:** Hann windows reused across FFT computations
2. **Frequency Array Cache:** RFFT frequency arrays cached by (length, sr) key
3. **Thread-Local LUFS Meters:** Per-thread meter instances avoid contention
4. **Polyphase Resampling:** GCD optimization to minimize filter order
5. **Stride Tricks:** `np.lib.stride_tricks.as_strided()` for frame RMS computation without copying
6. **In-Memory Encoding:** BytesIO buffer avoids disk I/O during encoding
7. **Back-Pressure Control:** `max_inflight` limits prevents unbounded memory growth
8. **Double-Check Locking:** Cache initialization with double-check pattern

---

### 11. COMMAND-LINE CONFIGURATION

**Key Audio Processing Parameters:**
```bash
--output-format {flac,wav}        # Output encoding format
--chunk-long / --no-chunk-long    # Enable chunking (default: enabled)
--chunk-min-sec FLOAT             # Minimum chunk length (default: 6.0)
--chunk-max-sec FLOAT             # Maximum chunk length (default: 10.0)
--chunk-threshold-sec FLOAT       # Audio duration threshold to trigger chunking (default: 12.0)
--chunk-all-buckets               # Apply chunking to high-rate buckets (default: full segments)
--silence-snap / --no-silence-snap # Snap to low-energy points (default: enabled)
--silence-frame-ms FLOAT          # Frame length for RMS computation (default: 50ms)
--silence-hop-ms FLOAT            # Hop size for frame RMS (default: 10ms)
--silence-search-sec FLOAT        # Search window around target chunk boundary (default: 0.5s)
--silence-percentile FLOAT        # Percentile for silence threshold (default: 20.0)
--silence-db-drop FLOAT           # dB drop to trigger silence boundary (default: 20.0)
--profile-timing                  # Enable detailed profiling
--segment-workers INT             # Segment processor threads (default: 8)
--segment-max-inflight INT        # Max in-flight segment tasks (default: 128 or workers*2)
```

---

**File:** `/home/ubuntu/maya3_prepare/scripts/prepare_shards.py`

This is a sophisticated, production-quality data pipeline with well-instrumented timing, thread-safe caching, and careful back-pressure management for processing large-scale audio datasets.