  1000→        help="Split long segments into 6-10s windows for the 16k bucket.",
  1001→    )
  1002→    parser.add_argument(
  1003→        "--no-chunk-long",
  1004→        dest="chunk_long",
  1005→        action="store_false",
  1006→        help="Disable chunking long segments.",
  1007→    )
  1008→    parser.add_argument("--chunk-min-sec", type=float, default=6.0)
  1009→    parser.add_argument("--chunk-max-sec", type=float, default=10.0)
  1010→    parser.add_argument("--chunk-threshold-sec", type=float, default=12.0)
  1011→    parser.add_argument(
  1012→        "--chunk-all-buckets",
  1013→        action="store_true",
  1014→        help="Also chunk high-rate buckets (default: keep full segments).",
  1015→    )
  1016→    parser.add_argument(
  1017→        "--silence-snap",
  1018→        dest="silence_snap",
  1019→        action="store_true",
  1020→        default=True,
  1021→        help="Snap chunk boundaries to low-energy points when possible.",
  1022→    )
  1023→    parser.add_argument(
  1024→        "--no-silence-snap",
  1025→        dest="silence_snap",
  1026→        action="store_false",
  1027→        help="Disable silence-aware chunk boundaries.",
  1028→    )
  1029→    parser.add_argument("--silence-frame-ms", type=float, default=50.0)
  1030→    parser.add_argument("--silence-hop-ms", type=float, default=10.0)
  1031→    parser.add_argument("--silence-search-sec", type=float, default=0.5)
  1032→    parser.add_argument("--silence-percentile", type=float, default=20.0)
  1033→    parser.add_argument("--silence-db-drop", type=float, default=20.0)
  1034→    parser.add_argument("--upload", action="store_true")
  1035→    parser.add_argument("--verify-upload", action="store_true")
  1036→    parser.add_argument("--create-prefixes", action="store_true")
  1037→    parser.add_argument("--r2-subprefix", default="")
  1038→    parser.add_argument(
  1039→        "--profile-timing",
  1040→        action="store_true",
  1041→        help="Enable detailed timing instrumentation for profiling.",
  1042→    )
  1043→    parser.add_argument(
  1044→        "--output-format",
  1045→        choices=["flac", "wav"],
  1046→        default="flac",
  1047→        help="Output audio format for shards (default: flac).",
  1048→    )
  1049→    parser.add_argument(
  1050→        "--gpu",
  1051→        action="store_true",
  1052→        help="Use GPU-accelerated pipeline (requires torch, torchaudio).",
  1053→    )
  1054→    parser.add_argument(
  1055→        "--gpu-batch-size",
  1056→        type=int,
  1057→        default=32,
  1058→        help="Batch size for GPU processing (default: 32).",
  1059→    )
  1060→    parser.add_argument(
  1061→        "--gpu-decode-workers",
  1062→        type=int,
  1063→        default=8,
  1064→        help="Number of parallel decode workers for GPU pipeline (default: 8).",
  1065→    )
  1066→    parser.add_argument(
  1067→        "--gpu-encode-workers",
  1068→        type=int,
  1069→        default=8,
  1070→        help="Number of parallel encode workers for GPU pipeline (default: 8).",
  1071→    )
  1072→    return parser.parse_args()
  1073→
  1074→
  1075→def main() -> None:
  1076→    global _timing_enabled
  1077→    args = parse_args()
  1078→    _timing_enabled = args.profile_timing
  1079→    env = load_env(ENV_PATH)
  1080→    db_url = env.get("DATABASE_URL")
  1081→    if not db_url:
  1082→        raise SystemExit("DATABASE_URL not found in .env")
  1083→
  1084→    cleaned_bucket = env.get("R2_ClEANED_BUCKET") or env.get("R2_CLEANED_BUCKET")
  1085→    shard_bucket = env.get("R2_BUCKET_SHARDS")
  1086→    prefix = env.get("R2_PREFIX")
  1087→    if not cleaned_bucket:
  1088→        raise SystemExit("R2 cleaned bucket not found in .env")
  1089→
  1090→    if args.upload and not shard_bucket:
  1091→        raise SystemExit("R2_BUCKET_SHARDS not found in .env")
  1092→
  1093→    if args.chunk_min_sec <= 0 or args.chunk_max_sec <= 0:
  1094→        raise SystemExit("Chunk durations must be positive.")
  1095→    if args.chunk_min_sec > args.chunk_max_sec:
  1096→        raise SystemExit("chunk-min-sec must be <= chunk-max-sec.")
  1097→    if args.chunk_threshold_sec < args.chunk_min_sec:
  1098→        raise SystemExit("chunk-threshold-sec must be >= chunk-min-sec.")
  1099→    if args.silence_frame_ms <= 0 or args.silence_hop_ms <= 0:
  1100→        raise SystemExit("Silence frame/hop must be positive.")
  1101→    if args.silence_search_sec <= 0:
  1102→        raise SystemExit("silence-search-sec must be positive.")
  1103→    if not 0.0 < args.silence_percentile < 100.0:
  1104→        raise SystemExit("silence-percentile must be between 0 and 100.")
  1105→    if args.silence_db_drop <= 0:
  1106→        raise SystemExit("silence-db-drop must be positive.")
  1107→
  1108→    if args.partition_count < 1:
  1109→        raise SystemExit("partition-count must be >= 1.")
  1110→    if not 0 <= args.partition_index < args.partition_count:
  1111→        raise SystemExit("partition-index must be between 0 and partition-count - 1.")
  1112→
  1113→    output_format = args.output_format
  1114→    audio_ext = "wav" if output_format == "wav" else "flac"
  1115→
  1116→    partition_tag = ""
  1117→    if args.partition_count > 1:
  1118→        partition_tag = f"p{args.partition_index:02d}"
  1119→
  1120→    chunk_cfg = ChunkConfig(
  1121→        enabled=args.chunk_long,
  1122→        min_sec=args.chunk_min_sec,
  1123→        max_sec=args.chunk_max_sec,
  1124→        threshold_sec=args.chunk_threshold_sec,
  1125→        all_buckets=args.chunk_all_buckets,
  1126→    )
  1127→    silence_cfg = SilenceConfig(
  1128→        enabled=args.silence_snap,
  1129→        frame_sec=args.silence_frame_ms / 1000.0,
  1130→        hop_sec=args.silence_hop_ms / 1000.0,
  1131→        search_sec=args.silence_search_sec,
  1132→        percentile=args.silence_percentile,
  1133→        db_drop=args.silence_db_drop,
  1134→    )
  1135→    language = args.language.strip().lower()
  1136→    with timer("db_fetch_records"):
  1137→        records = fetch_video_records(
  1138→            db_url,
  1139→            language,
  1140→            args.limit,
  1141→            args.partition_count,
  1142→            args.partition_index,
  1143→        )
  1144→    if not records:
  1145→        raise SystemExit(f"No records found for language={language}")
  1146→
  1147→    s3 = make_r2_client(env)
  1148→    if args.create_prefixes and args.upload:
  1149→        create_prefix_placeholders(

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

        help="Split long segments into 6-10s windows for the 16k bucket.",
    )
    parser.add_argument(
        "--no-chunk-long",
        dest="chunk_long",
        action="store_false",
        help="Disable chunking long segments.",
    )
    parser.add_argument("--chunk-min-sec", type=float, default=6.0)
    parser.add_argument("--chunk-max-sec", type=float, default=10.0)
    parser.add_argument("--chunk-threshold-sec", type=float, default=12.0)
    parser.add_argument(
        "--chunk-all-buckets",
        action="store_true",
        help="Also chunk high-rate buckets (default: keep full segments).",
    )
    parser.add_argument(
        "--silence-snap",
        dest="silence_snap",
        action="store_true",
        default=True,
        help="Snap chunk boundaries to low-energy points when possible.",
    )
    parser.add_argument(
        "--no-silence-snap",
        dest="silence_snap",
        action="store_false",
        help="Disable silence-aware chunk boundaries.",
    )
    parser.add_argument("--silence-frame-ms", type=float, default=50.0)
    parser.add_argument("--silence-hop-ms", type=float, default=10.0)
    parser.add_argument("--silence-search-sec", type=float, default=0.5)
    parser.add_argument("--silence-percentile", type=float, default=20.0)
    parser.add_argument("--silence-db-drop", type=float, default=20.0)
    parser.add_argument("--upload", action="store_true")
    parser.add_argument("--verify-upload", action="store_true")
    parser.add_argument("--create-prefixes", action="store_true")
    parser.add_argument("--r2-subprefix", default="")
    parser.add_argument(
        "--profile-timing",
        action="store_true",
        help="Enable detailed timing instrumentation for profiling.",
    )
    parser.add_argument(
        "--output-format",
        choices=["flac", "wav"],
        default="flac",
        help="Output audio format for shards (default: flac).",
    )
    parser.add_argument(
        "--gpu",
        action="store_true",
        help="Use GPU-accelerated pipeline (requires torch, torchaudio).",
    )
    parser.add_argument(
        "--gpu-batch-size",
        type=int,
        default=32,
        help="Batch size for GPU processing (default: 32).",
    )
    parser.add_argument(
        "--gpu-decode-workers",
        type=int,
        default=8,
        help="Number of parallel decode workers for GPU pipeline (default: 8).",
    )
    parser.add_argument(
        "--gpu-encode-workers",
        type=int,
        default=8,
        help="Number of parallel encode workers for GPU pipeline (default: 8).",
    )
    return parser.parse_args()

def main() -> None:
    global _timing_enabled
    args = parse_args()
    _timing_enabled = args.profile_timing
    env = load_env(ENV_PATH)
    db_url = env.get("DATABASE_URL")
    if not db_url:
        raise SystemExit("DATABASE_URL not found in .env")

    cleaned_bucket = env.get("R2_ClEANED_BUCKET") or env.get("R2_CLEANED_BUCKET")
    shard_bucket = env.get("R2_BUCKET_SHARDS")
    prefix = env.get("R2_PREFIX")
    if not cleaned_bucket:
        raise SystemExit("R2 cleaned bucket not found in .env")

    if args.upload and not shard_bucket:
        raise SystemExit("R2_BUCKET_SHARDS not found in .env")

    if args.chunk_min_sec <= 0 or args.chunk_max_sec <= 0:
        raise SystemExit("Chunk durations must be positive.")
    if args.chunk_min_sec > args.chunk_max_sec:
        raise SystemExit("chunk-min-sec must be <= chunk-max-sec.")
    if args.chunk_threshold_sec < args.chunk_min_sec:
        raise SystemExit("chunk-threshold-sec must be >= chunk-min-sec.")
    if args.silence_frame_ms <= 0 or args.silence_hop_ms <= 0:
        raise SystemExit("Silence frame/hop must be positive.")
    if args.silence_search_sec <= 0:
        raise SystemExit("silence-search-sec must be positive.")
    if not 0.0 < args.silence_percentile < 100.0:
        raise SystemExit("silence-percentile must be between 0 and 100.")
    if args.silence_db_drop <= 0:
        raise SystemExit("silence-db-drop must be positive.")

    if args.partition_count < 1:
        raise SystemExit("partition-count must be >= 1.")
    if not 0 <= args.partition_index < args.partition_count:
        raise SystemExit("partition-index must be between 0 and partition-count - 1.")

    output_format = args.output_format
    audio_ext = "wav" if output_format == "wav" else "flac"

    partition_tag = ""
    if args.partition_count > 1:
        partition_tag = f"p{args.partition_index:02d}"

    chunk_cfg = ChunkConfig(
        enabled=args.chunk_long,
        min_sec=args.chunk_min_sec,
        max_sec=args.chunk_max_sec,
        threshold_sec=args.chunk_threshold_sec,
        all_buckets=args.chunk_all_buckets,
    )
    silence_cfg = SilenceConfig(
        enabled=args.silence_snap,
        frame_sec=args.silence_frame_ms / 1000.0,
        hop_sec=args.silence_hop_ms / 1000.0,
        search_sec=args.silence_search_sec,
        percentile=args.silence_percentile,
        db_drop=args.silence_db_drop,
    )
    language = args.language.strip().lower()
    with timer("db_fetch_records"):
        records = fetch_video_records(
            db_url,
            language,
            args.limit,
            args.partition_count,
            args.partition_index,
        )
    if not records:
        raise SystemExit(f"No records found for language={language}")

    s3 = make_r2_client(env)
    if args.create_prefixes and args.upload:
        create_prefix_placeholders(


Perfect! I now have a comprehensive understanding of the codebase. Let me create a detailed summary report.

## Comprehensive Audio Processing Codebase Analysis

I've thoroughly explored the audio processing codebase. Here's a comprehensive summary:

---

### **1. OVERALL PROJECT STRUCTURE**

**Location:** `/home/ubuntu/maya3_prepare/`

**Key Components:**
```
├── scripts/
│   ├── prepare_shards.py           (1,589 lines) - Main pipeline
│   ├── fast_audio_pipeline.py      (613 lines)  - GPU-accelerated v1
│   ├── fast_audio_pipeline_v2.py   (447 lines)  - GPU-accelerated v2 (parallel)
│   ├── benchmark_gpu.py            - Performance benchmarking
│   ├── bulk_download.py            - Download management
│   ├── fetch_languages.py          - Supabase language lookup
│   ├── PERFORMANCE_ANALYSIS.md     - Detailed bottleneck analysis
│   └── benchmark_results.json      - Benchmark data
├── X-Codec-2.0/                    - Codec training framework
│   ├── lightning_module.py         - Codec training with W2V-BERT
│   ├── data_module.py              - DataLoader configuration
│   ├── inference.py                - Codec inference
│   └── train.py                    - Training entrypoint
├── tts/                            - TTS codec infrastructure
├── strategy.md                     - 12-language training strategy
├── objective.md                    - Data preparation objectives
└── optimisations.md                - Optimization roadmap (45.8 days → <24h)
```

**Language Distribution:**
- 12 Indic languages + English
- ~507k videos, ~149k usable hours
- Most data: Telugu (13.2%), Malayalam (12.2%), English (12%), Hindi (11.9%)

---

### **2. MAIN PROCESSING PIPELINE SCRIPTS**

#### **Core: `prepare_shards.py` (1,589 lines)**

**Purpose:** Convert YouTube FLAC segments from Cloudflare R2 → WebDataset shards with computed metrics

**Main Functions:**

1. **Data Loading**
   - `fetch_video_records()`: Query PostgreSQL for videos by language
   - Support for partitioned processing (distribute across workers)

2. **Audio Metrics Computation**
   - `compute_audio_metrics()`: Main metrics function
     - RMS/loudness computation
     - **pyloudnorm LUFS** (integrated loudness, ITU-R BS.1770)
     - Spectral analysis (rolloff, effective Nyquist, HF energy)
     - SNR proxy, clipping fraction, DC offset
   - `compute_spectral_metrics()`: FFT-based bandwidth detection
   - `compute_quality_score()`: Weighted scoring for sampling

3. **Segment Processing**
   - `process_segment()`: Main per-segment handler
     - FLAC decode → chunk → metrics → resample (16k/24k/32k/44.1k) → WAV encode
   - `make_chunk_windows()`: Split long segments (>12s) into 6-10s windows
   - Silence-aware chunking: Snap boundaries to low-energy points

4. **Sharding**
   - `write_shard()`: Pack segments into tar files (~5GB each)
   - Per-shard JSONL metadata sidecar
   - Train/val splitting (99%/1% by video_id)

5. **R2 Integration**
   - `make_r2_client()`: Boto3 S3-compatible client
   - Multipart upload, retry logic
   - Prefix management for language/SR buckets

**Output Format:**
```
processed_data/
├── manifests/
│   ├── telugu_train.jsonl
│   └── telugu_val.jsonl
├── telugu_16/
│   ├── shard-000000.tar
│   ├── shard-000000.jsonl
│   └── ...
├── telugu_24/, telugu_32/, telugu_44/  (conditional on bandwidth)
```

**Per-Sample Metadata (JSONL):**
```json
{
  "id": "videoID__SPEAKER_00_0001_c00",
  "video_id": "---g_VL7ySo",
  "language": "telugu",
  "duration_sec": 6.5,
  "sample_rate": 16000,
  "loudness_lufs": -23.5,
  "loudness_rms_db": -18.2,
  "clipping_fraction": 0.0,
  "effective_nyquist_hz": 10500.0,
  "native_sr_bucket": "16-24k",
  "is_truly_wideband": false,
  "quality_score": 0.82,
  "duration_bucket": "6-12s",
  "sampling_weight": 1.0
}
```

---

### **3. HOW FLAC DECODING, LUFS, RESAMPLING, AND WAV ENCODING ARE IMPLEMENTED**

#### **A. FLAC Decoding (31% of processing time)**

**Current Implementation:**
```python
# prepare_shards.py lines 587-591
with timer("flac_decode"):
    try:
        audio, sr = sf.read(io.BytesIO(segment_bytes), dtype="float32")
    except Exception:
        return outputs, stats
```

- **Library:** `soundfile` (libsndfile C library binding)
- **Parallelization:** ThreadPoolExecutor with 8-12 workers (I/O bound)
- **Limitations:** Can't use GPU; network I/O blocks threads

#### **B. LUFS Computation (45% of processing time) - PRIMARY BOTTLENECK**

**Implementation:**
```python
# prepare_shards.py lines 328-336
with timer("audio_metrics_lufs"):
    loudness_lufs = -100.0
    if pyln is not None:
        try:
            meter = get_lufs_meter(sr)  # Thread-local per-SR meter
            if meter is not None:
                loudness_lufs = float(meter.integrated_loudness(audio))
        except Exception:
            loudness_lufs = -100.0
```

**Algorithm:** ITU-R BS.1770-4 (EBU R128)
1. K-weighting filters (cascaded IIR biquads) → 100 Hz high-pass + 2 kHz high-shelf
2. Mean square computation over 400ms blocks (75% overlap)
3. Gating: Silence gate (-70 LUFS), relative gate (-10 LUFS below overall)
4. Integration: Average across valid blocks

**Performance:**
- CPU: 26.1 ms per segment (pure Python sequential)
- **Pure Python implementation, releases GIL but can't parallelize**
- Thread-local caching with `get_lufs_meter()` for reuse

**Optimization Opportunities:**
1. Skip LUFS for quality filtering (use RMS approximation)
2. GPU implementation with PyTorch (10-32x speedup with batching)
3. C extension or faster library (ffmpeg-normalize)

#### **C. Resampling (0.9% GPU time, but 12ms CPU time)**

**Current Implementation:**
```python
# prepare_shards.py lines 370-374
def resample_audio(audio, src_sr, dst_sr):
    with timer("cpu_resample"):
        gcd = math.gcd(src_sr, dst_sr)
        up = dst_sr // gcd
        down = src_sr // gcd
        return signal.resample_poly(audio, up, down).astype(np.float32)
```

- **Library:** SciPy `resample_poly` (polyphase resampling)
- **CPU Path:** Sequential per-target rate
- **GPU Path** (fast_audio_pipeline.py):
  ```python
  def resample(self, audio, src_sr, dst_sr):
      resampler = torchaudio.transforms.Resample(src_sr, dst_sr)
      audio_t = torch.from_numpy(audio).unsqueeze(0).to(device)
      resampled_t = resampler(audio_t)
      return resampled_t.squeeze(0).cpu().numpy()
  ```
  - **Bottleneck:** CPU↔GPU memory transfer (small batches, not amortized)
  - **Real benefit:** Only 2% improvement over CPU on A100

#### **D. WAV Encoding (22.8% of processing time)**

**Implementation:**
```python
# prepare_shards.py lines 378-384
def encode_audio(audio, sr, output_format):
    with timer("audio_encode"):
        buf = io.BytesIO()
        if output_format == "wav":
            sf.write(buf, audio, sr, format="WAV", subtype="PCM_16")
        else:
            sf.write(buf, audio, sr, format="FLAC", subtype="PCM_16")
        return buf.getvalue()
```

- **Library:** `soundfile` (libsndfile)
- **Parallelization:** ThreadPoolExecutor with 8-12 workers
- **Format:** PCM_16 (16-bit signed int)
- **No GPU acceleration available** (CPU-bound encoding)

---

### **4. CONFIGURATION & BENCHMARK SCRIPTS**

#### **Configuration Parameters**

**Chunking Config:**
- `chunk_long`: Enable/disable (default: True)
- `chunk_min_sec`: 6.0 (minimum output segment)
- `chunk_max_sec`: 10.0 (maximum output segment)
- `chunk_threshold_sec`: 12.0 (split segments longer than this)
- `chunk_all_buckets`: False (only chunk 16k; keep high-rates full)

**Silence Config:**
- `silence_snap`: True (snap boundaries to silence)
- `frame_sec`: 0.05 (50ms frame for RMS computation)
- `hop_sec`: 0.01 (10ms hop)
- `search_sec`: 0.5 (search ±0.5s for silence)
- `percentile`: 20.0 (use 20th percentile RMS as threshold)
- `db_drop`: 20.0 (cut if 20dB quieter than median)

**Output Config:**
- `--output-format`: "wav" or "flac" (default: flac for R2 storage)
- `--partition-count`, `--partition-index`: Horizontal scaling

#### **Benchmark Scripts**

**File:** `benchmark_gpu.py` (A100-80GB testing)

**Tested Configurations:**
- Batch sizes: 32, 64, 128, 256
- Decode workers: 8, 12, 16
- Encode workers: 8, 12, 16

**Best Configuration (from benchmark_results.json):**
```
Batch size: 256
Decode workers: 12
Encode workers: 12
Throughput: 29.13 seg/s
GPU memory: 35.2 MB (essentially idle)
```

**Key Finding:** GPU memory usage stays ~35MB across all configs
- No batching benefit
- GPU underutilized due to CPU-bound metrics (LUFS)

---

### **5. PERFORMANCE BOTTLENECK ANALYSIS**

#### **Per-Operation Breakdown (Single Segment, 48kHz):**

| Operation | Time | % of Total | Notes |
|-----------|------|-----------|-------|
| **FLAC Decode** | 17.9ms | **31.0%** | Parallelizable (I/O bound) |
| **LUFS+Spectral Metrics** | 26.1ms | **45.3%** | CPU-bound, pyloudnorm sequential |
| **GPU Resample (4 rates)** | 0.5ms | 0.9% | Transfer overhead dominates |
| **WAV Encode (4 rates)** | 13.1ms | **22.8%** | Parallelizable (I/O bound) |
| **Total** | **57.7ms** | 100% | **17.3 seg/s single-threaded** |

#### **Real Throughput with Current Parallelization:**
- **26.1 seg/s** (7.56 videos/min) on A100 + 16 vCPUs
- **Parallelization ceiling:** Decode (8-12 workers) + LUFS (sequential) + Encode (8-12 workers)
- **Why GPU is idle:** LUFS computation (45%) is pure Python, releases GIL but can't be GPU-accelerated with current code

#### **Scaling Analysis:**
- **500k videos:** 1,100 hours (46 days) at 26.1 seg/s
- **Target:** 500k in 24 hours = 347 videos/min = 5.8× speedup needed
- **With optimization:** GPU LUFS (11× speedup single node) + 8-node cluster = **8-96× speedup**

---

### **6. KEY FILES & LOCATIONS**

**Core Processing:**
- `/home/ubuntu/maya3_prepare/scripts/prepare_shards.py` - Main pipeline (1,589 lines)

**GPU Acceleration:**
- `/home/ubuntu/maya3_prepare/scripts/fast_audio_pipeline.py` - Equivalent batch processor
- `/home/ubuntu/maya3_prepare/scripts/fast_audio_pipeline_v2.py` - Fully parallelized v2

**Codec Training:**
- `/home/ubuntu/maya3_prepare/X-Codec-2.0/lightning_module.py` - Model training
- `/home/ubuntu/maya3_prepare/X-Codec-2.0/data_module.py` - Data loading

**Documentation:**
- `/home/ubuntu/maya3_prepare/strategy.md` - Complete 12-language strategy
- `/home/ubuntu/maya3_prepare/objective.md` - Data prep requirements
- `/home/ubuntu/maya3_prepare/optimisations.md` - Optimization roadmap
- `/home/ubuntu/maya3_prepare/scripts/PERFORMANCE_ANALYSIS.md` - Detailed bottleneck report

**Benchmarking:**
- `/home/ubuntu/maya3_prepare/scripts/benchmark_gpu.py` - A100 benchmark suite
- `/home/ubuntu/maya3_prepare/scripts/benchmark_results.json` - Results (best: 29.13 seg/s)

---

### **7. CRITICAL INSIGHTS**

1. **GPU is 95% idle** - Only resampling uses GPU (0.9% of time), and even then it's slower than CPU due to transfer overhead

2. **pyloudnorm LUFS is the killer** (45% of time)
   - Pure Python ITU-R BS.1770 implementation
   - Releases GIL but can't parallelize or GPU-accelerate in current code
   - Recommendation: Batch GPU LUFS + spectral analysis for 10-32× speedup

3. **Parallelization works well** for I/O-bound ops:
   - Decode: 31% of time, scales with workers (currently 8-12)
   - Encode: 22.8% of time, scales with workers (currently 8-12)
   - Sequential LUFS (45%) is the floor

4. **Horizontal scaling is the fast path:**
   - Linear scaling up to 8-16 nodes with no inter-node dependencies
   - 500k videos in 3-6 days on 8× c6i.4xlarge (CPU-only, cheaper)
   - 500k videos in 24 hours on 8× g5.4xlarge + GPU LUFS optimization

5. **Data format considerations:**
   - Input: FLAC from YouTube (compressed)
   - Output: WAV (uncompressed) or FLAC (compressed)
   - Storing as FLAC saves bandwidth; decoding costs little vs network savings

---

### **8. ARCHITECTURE SUMMARY DIAGRAM**

```
PostgreSQL Supabase (Video Index)
        ↓
Download from Cloudflare R2 (FLAC segments)
        ↓
ThreadPool Decode (8-12 workers) [31% time] ←── I/O bound
        ↓
Sequential pyloudnorm LUFS Metrics [45% time] ←── CPU-bound BOTTLENECK
(+ Spectral FFT + RMS + SNR computation)
        ↓
Silence-aware Chunking (6-10s windows)
        ↓
Resample to target SR (16k/24k/32k/44.1k) [0.9% time on GPU, else 12ms CPU]
        ↓
ThreadPool Encode WAV (8-12 workers) [22.8% time] ←── I/O bound
        ↓
Pack into tar shards + JSONL metadata
        ↓
Upload to R2 Cloudflare (WebDataset format)
```

**Total Throughput:** 26-29 seg/s (7.5-8.5 videos/min) on A100 + 16 vCPU

This codebase is production-ready but heavily limited by the sequential LUFS bottleneck. The optimizations.md roadmap clearly identifies GPU LUFS batching as the key to achieving sub-24-hour processing for 500k videos.