     1→#!/usr/bin/env python3
     2→"""GPU Pipeline Performance Benchmark for A100-80GB.
     3→
     4→Tests various configurations to find optimal settings:
     5→- Batch sizes: 16, 32, 64, 128, 256
     6→- Decode workers: 4, 8, 12, 16
     7→- Encode workers: 4, 8, 12, 16
     8→
     9→Measures:
    10→- Segments/second throughput
    11→- GPU utilization
    12→- Memory usage
    13→- Bottleneck identification
    14→"""
    15→
    16→import argparse
    17→import io
    18→import json
    19→import os
    20→import sys
    21→import tarfile
    22→import time
    23→from concurrent.futures import ThreadPoolExecutor
    24→from dataclasses import dataclass
    25→from pathlib import Path
    26→from typing import Optional
    27→
    28→import numpy as np
    29→import soundfile as sf
    30→
    31→# Add current dir to path for imports
    32→sys.path.insert(0, str(Path(__file__).parent))
    33→
    34→from prepare_shards import (
    35→    ChunkConfig, SilenceConfig, make_segment_seed, parse_segment_filename
    36→)
    37→from fast_audio_pipeline import EquivalentBatchProcessor, SegmentInput, OutputSegment
    38→
    39→try:
    40→    import torch
    41→    HAS_TORCH = True
    42→except ImportError:
    43→    HAS_TORCH = False
    44→
    45→
    46→@dataclass
    47→class BenchmarkResult:
    48→    config_name: str
    49→    batch_size: int
    50→    decode_workers: int
    51→    encode_workers: int
    52→    total_segments: int
    53→    total_outputs: int
    54→    elapsed_sec: float
    55→    segments_per_sec: float
    56→    outputs_per_sec: float
    57→    gpu_memory_mb: float
    58→    stats: dict
    59→
    60→
    61→def get_gpu_memory_mb() -> float:
    62→    """Get current GPU memory usage in MB."""
    63→    if not HAS_TORCH or not torch.cuda.is_available():
    64→        return 0.0
    65→    return torch.cuda.memory_allocated() / 1024 / 1024
    66→
    67→
    68→def load_segments_from_tar(tar_path: Path, video_id: str, language: str, seed: int) -> list[SegmentInput]:
    69→    """Load segments from a tar file."""
    70→    segments = []
    71→    try:
    72→        with tarfile.open(tar_path, "r") as tf:
    73→            meta = json.loads(tf.extractfile(f"{video_id}/metadata.json").read())
    74→            usable_segments = [s for s in meta.get("segments", []) if s.get("status") == "usable"]
    75→            segment_map = {
    76→                (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
    77→                for seg in usable_segments
    78→            }
    79→
    80→            members = [m for m in tf.getmembers() if m.name.endswith(".flac")]
    81→
    82→            for member in members:
    83→                parsed = parse_segment_filename(member.name)
    84→                if not parsed:
    85→                    continue
    86→                speaker, start_time = parsed
    87→                seg_meta = segment_map.get((speaker, round(start_time, 2)))
    88→                if not seg_meta:
    89→                    matches = [
    90→                        s for s in usable_segments
    91→                        if s.get("speaker") == speaker
    92→                        and abs(s.get("start", 0.0) - start_time) < 0.1
    93→                    ]
    94→                    if not matches:
    95→                        continue
    96→                    seg_meta = matches[0]
    97→
    98→                segment_bytes = tf.extractfile(member).read()
    99→                filename_parts = member.name.split("_")
   100→                try:
   101→                    index = int(filename_parts[-2])
   102→                except (ValueError, IndexError):
   103→                    index = 0
   104→                rng_seed = make_segment_seed(seed, video_id, speaker, index)
   105→
   106→                segments.append(SegmentInput(
   107→                    video_id=video_id,
   108→                    language=language,
   109→                    segment_bytes=segment_bytes,
   110→                    segment_meta=seg_meta,
   111→                    speaker=speaker,
   112→                    index=index,
   113→                    is_val=False,
   114→                    rng_seed=rng_seed,
   115→                ))
   116→    except Exception as e:
   117→        print(f"Error loading {tar_path}: {e}")
   118→
   119→    return segments
   120→
   121→
   122→def generate_synthetic_segments(count: int, duration_sec: float = 10.0, sr: int = 48000) -> list[SegmentInput]:
   123→    """Generate synthetic test segments."""
   124→    segments = []
   125→    num_samples = int(sr * duration_sec)
   126→    t = np.linspace(0, duration_sec, num_samples, dtype=np.float32)
   127→
   128→    for i in range(count):
   129→        # Generate varied audio
   130→        freq = 200 + (i % 20) * 100
   131→        audio = (
   132→            np.sin(2 * np.pi * freq * t) * 0.3 +
   133→            np.sin(2 * np.pi * 1000 * t) * 0.2 +
   134→            np.sin(2 * np.pi * 5000 * t) * 0.1 +
   135→            np.random.randn(num_samples).astype(np.float32) * 0.02
   136→        ).astype(np.float32)
   137→
   138→        buf = io.BytesIO()
   139→        sf.write(buf, audio, sr, format="FLAC")
   140→        flac_bytes = buf.getvalue()
   141→
   142→        segments.append(SegmentInput(
   143→            video_id=f"synthetic_{i}",
   144→            language="english",
   145→            segment_bytes=flac_bytes,
   146→            segment_meta={"start": 0.0, "end": duration_sec, "original_start": 0.0, "original_end": duration_sec},
   147→            speaker="SPEAKER_00",
   148→            index=i,
   149→            is_val=False,
   150→            rng_seed=42 + i,

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

#!/usr/bin/env python3
"""GPU Pipeline Performance Benchmark for A100-80GB.

Tests various configurations to find optimal settings:
- Batch sizes: 16, 32, 64, 128, 256
- Decode workers: 4, 8, 12, 16
- Encode workers: 4, 8, 12, 16

Measures:
- Segments/second throughput
- GPU utilization
- Memory usage
- Bottleneck identification
"""

import argparse
import io
import json
import os
import sys
import tarfile
import time
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from pathlib import Path
from typing import Optional

import numpy as np
import soundfile as sf

# Add current dir to path for imports
sys.path.insert(0, str(Path(__file__).parent))

from prepare_shards import (
    ChunkConfig, SilenceConfig, make_segment_seed, parse_segment_filename
)
from fast_audio_pipeline import EquivalentBatchProcessor, SegmentInput, OutputSegment

try:
    import torch
    HAS_TORCH = True
except ImportError:
    HAS_TORCH = False

@dataclass
class BenchmarkResult:
    config_name: str
    batch_size: int
    decode_workers: int
    encode_workers: int
    total_segments: int
    total_outputs: int
    elapsed_sec: float
    segments_per_sec: float
    outputs_per_sec: float
    gpu_memory_mb: float
    stats: dict

def get_gpu_memory_mb() -> float:
    """Get current GPU memory usage in MB."""
    if not HAS_TORCH or not torch.cuda.is_available():
        return 0.0
    return torch.cuda.memory_allocated() / 1024 / 1024

def load_segments_from_tar(tar_path: Path, video_id: str, language: str, seed: int) -> list[SegmentInput]:
    """Load segments from a tar file."""
    segments = []
    try:
        with tarfile.open(tar_path, "r") as tf:
            meta = json.loads(tf.extractfile(f"{video_id}/metadata.json").read())
            usable_segments = [s for s in meta.get("segments", []) if s.get("status") == "usable"]
            segment_map = {
                (seg.get("speaker"), round(seg.get("start", 0.0), 2)): seg
                for seg in usable_segments
            }

            members = [m for m in tf.getmembers() if m.name.endswith(".flac")]

            for member in members:
                parsed = parse_segment_filename(member.name)
                if not parsed:
                    continue
                speaker, start_time = parsed
                seg_meta = segment_map.get((speaker, round(start_time, 2)))
                if not seg_meta:
                    matches = [
                        s for s in usable_segments
                        if s.get("speaker") == speaker
                        and abs(s.get("start", 0.0) - start_time) < 0.1
                    ]
                    if not matches:
                        continue
                    seg_meta = matches[0]

                segment_bytes = tf.extractfile(member).read()
                filename_parts = member.name.split("_")
                try:
                    index = int(filename_parts[-2])
                except (ValueError, IndexError):
                    index = 0
                rng_seed = make_segment_seed(seed, video_id, speaker, index)

                segments.append(SegmentInput(
                    video_id=video_id,
                    language=language,
                    segment_bytes=segment_bytes,
                    segment_meta=seg_meta,
                    speaker=speaker,
                    index=index,
                    is_val=False,
                    rng_seed=rng_seed,
                ))
    except Exception as e:
        print(f"Error loading {tar_path}: {e}")

    return segments

def generate_synthetic_segments(count: int, duration_sec: float = 10.0, sr: int = 48000) -> list[SegmentInput]:
    """Generate synthetic test segments."""
    segments = []
    num_samples = int(sr * duration_sec)
    t = np.linspace(0, duration_sec, num_samples, dtype=np.float32)

    for i in range(count):
        # Generate varied audio
        freq = 200 + (i % 20) * 100
        audio = (
            np.sin(2 * np.pi * freq * t) * 0.3 +
            np.sin(2 * np.pi * 1000 * t) * 0.2 +
            np.sin(2 * np.pi * 5000 * t) * 0.1 +
            np.random.randn(num_samples).astype(np.float32) * 0.02
        ).astype(np.float32)

        buf = io.BytesIO()
        sf.write(buf, audio, sr, format="FLAC")
        flac_bytes = buf.getvalue()

        segments.append(SegmentInput(
            video_id=f"synthetic_{i}",
            language="english",
            segment_bytes=flac_bytes,
            segment_meta={"start": 0.0, "end": duration_sec, "original_start": 0.0, "original_end": duration_sec},
            speaker="SPEAKER_00",
            index=i,
            is_val=False,
            rng_seed=42 + i,


     1→# GPU Pipeline Performance Analysis for A100-80GB
     2→
     3→## Executive Summary
     4→
     5→**Current Performance:** ~26 seg/s, ~7.5 videos/min on A100-80GB with 16 vCPUs
     6→
     7→**Key Finding:** The A100 GPU is heavily underutilized. The bottleneck is CPU-bound metrics computation (pyloudnorm LUFS), not GPU operations.
     8→
     9→**Recommendation:** For cost optimization, consider using CPU-only instances (e.g., c6i.4xlarge) instead of expensive GPU instances. The GPU provides minimal benefit for this workload.
    10→
    11→---
    12→
    13→## Benchmark Results
    14→
    15→### Per-Operation Timing (Single Segment)
    16→
    17→| Operation | Time (ms) | % of Total |
    18→|-----------|-----------|------------|
    19→| FLAC Decode | 17.9 | 31.0% |
    20→| **Metrics (LUFS+Spectral)** | **26.1** | **45.3%** |
    21→| GPU Resample (×4 rates) | 0.5 | 0.9% |
    22→| WAV Encode (×4 rates) | 13.1 | 22.8% |
    23→| **Total** | **57.7** | 100% |
    24→
    25→**The GPU is only used for 0.9% of the processing time.**
    26→
    27→### Parallelization Effectiveness
    28→
    29→| Workers | Metrics-Only | Full Pipeline (GPU) | Full Pipeline (CPU) |
    30→|---------|-------------|---------------------|---------------------|
    31→| 4 | 76.3 seg/s | 51.2 seg/s | 42.3 seg/s |
    32→| 8 | 96.8 seg/s | 50.7 seg/s | 50.1 seg/s |
    33→| 12 | 94.4 seg/s | 50.8 seg/s | 48.6 seg/s |
    34→| 16 | 92.2 seg/s | 49.4 seg/s | 46.8 seg/s |
    35→
    36→**Key Insight:** GPU resampling provides only ~2% improvement over CPU resampling. The GPU is bottlenecked by:
    37→1. CPU↔GPU memory transfer overhead
    38→2. Serialized CUDA stream execution
    39→3. The small batch size (single segment at a time)
    40→
    41→### Real Data Test (Hindi, 20 videos)
    42→
    43→- Segments loaded: 4,142
    44→- Segments/video: 207
    45→- Throughput: **26.1 seg/s**
    46→- Videos/min: **7.56**
    47→- Dropped (too short): 87.7%
    48→- Outputs generated: 7,393
    49→
    50→---
    51→
    52→## Bottleneck Analysis
    53→
    54→### Primary Bottleneck: pyloudnorm LUFS (45% of time)
    55→
    56→The `compute_audio_metrics()` function calls `pyloudnorm.Meter.integrated_loudness()`, which is a pure Python implementation of ITU-R BS.1770.
    57→
    58→**Options to optimize:**
    59→1. **Skip LUFS for filtering** - If LUFS is only used for quality scoring, consider a simpler RMS-based approximation
    60→2. **Batch LUFS on GPU** - Implement ITU-R BS.1770 K-weighting filters in PyTorch
    61→3. **Use a faster LUFS library** - e.g., `ffmpeg-normalize` or custom C extension
    62→
    63→### Secondary Bottleneck: FLAC Decode (31% of time)
    64→
    65→Using `soundfile` (libsndfile) for FLAC decoding. Already parallelized.
    66→
    67→**Options:**
    68→1. Store audio as WAV in cleaned bucket (eliminates decode entirely)
    69→2. Use `av` library (PyAV) which may be faster for some formats
    70→
    71→### Tertiary Bottleneck: WAV Encode (23% of time)
    72→
    73→Already parallelized. Limited optimization potential.
    74→
    75→---
    76→
    77→## Optimal Configuration
    78→
    79→### For A100-80GB + 16 vCPUs
    80→
    81→```bash
    82→python prepare_shards.py hindi --gpu \
    83→    --gpu-batch-size 128 \
    84→    --gpu-decode-workers 12 \
    85→    --gpu-encode-workers 12 \
    86→    --output-format wav
    87→```
    88→
    89→| Parameter | Value | Rationale |
    90→|-----------|-------|-----------|
    91→| gpu-batch-size | 128 | Marginal impact; larger batches don't help |
    92→| gpu-decode-workers | 12 | Optimal for 16 vCPUs (leaves headroom) |
    93→| gpu-encode-workers | 12 | Same as decode |
    94→
    95→### Expected Throughput
    96→
    97→| Metric | Value |
    98→|--------|-------|
    99→| Segments/sec | 26-30 |
   100→| Videos/min | 7.5-8.5 |
   101→| 16k outputs/min | ~1,500 |
   102→
   103→---
   104→
   105→## Scaling Projections
   106→
   107→### Single A100 Node
   108→
   109→| Dataset Size | Time | Days |
   110→|-------------|------|------|
   111→| 10,000 videos | 22 hours | 0.9 |
   112→| 50,000 videos | 110 hours | 4.6 |
   113→| 100,000 videos | 220 hours | 9.2 |
   114→| 500,000 videos | 1,100 hours | 45.8 |
   115→
   116→### Horizontal Scaling (500k videos)
   117→
   118→| Nodes | Time | Days | Cost Efficiency |
   119→|-------|------|------|-----------------|
   120→| 1 | 1,100h | 46 days | Baseline |
   121→| 2 | 550h | 23 days | 2× |
   122→| 4 | 275h | 11 days | 4× |
   123→| 8 | 138h | 5.7 days | 8× |
   124→| 16 | 69h | 2.9 days | 16× |
   125→| 32 | 34h | 1.4 days | 32× |
   126→
   127→**Scaling is linear** - no bottleneck on shared resources (each node processes independent videos).
   128→
   129→---
   130→
   131→## Cost Optimization Recommendations
   132→
   133→### Option 1: Keep A100 (Current)
   134→
   135→- **Cost:** ~$3.50/hour (p4d.24xlarge spot) or ~$10/hour (on-demand)
   136→- **Throughput:** 7.5 videos/min
   137→- **GPU utilization:** <5%
   138→- **Verdict:** Inefficient; paying for unused GPU capacity
   139→
   140→### Option 2: CPU-Only Instances (Recommended)
   141→
   142→Replace A100 with CPU instances:
   143→
   144→| Instance | vCPUs | Cost/hr | Est. Throughput | $/video |
   145→|----------|-------|---------|-----------------|---------|
   146→| c6i.4xlarge | 16 | $0.68 | 6.5 vid/min | $0.0017 |
   147→| c6i.8xlarge | 32 | $1.36 | 10 vid/min | $0.0023 |
   148→| c7i.4xlarge | 16 | $0.71 | 6.5 vid/min | $0.0018 |
   149→
   150→**Savings:** 80-90% cost reduction vs A100 with similar throughput.
   151→
   152→### Option 3: Hybrid Approach
   153→
   154→Use A100 nodes only if future optimizations (batch GPU LUFS, batch GPU metrics) are implemented.
   155→
   156→---
   157→
   158→## Implementation Recommendations
   159→
   160→### Immediate (No Code Changes)
   161→
   162→1. **Use CPU instances** instead of A100 for cost efficiency
   163→2. **Scale horizontally** with 4-8 CPU nodes for faster completion
   164→3. **Use optimal config:** 12 decode workers, 12 encode workers, batch_size 128
   165→
   166→### Short-term (Code Changes)
   167→
   168→1. **Parallelize segment processing** - Current code processes segments sequentially after decode; fully parallelize the entire chain
   169→2. **Remove GPU path** - It provides minimal benefit and adds complexity
   170→
   171→### Long-term (Significant Changes)
   172→
   173→1. **Store cleaned audio as WAV** - Eliminates FLAC decode overhead (31% savings)
   174→2. **Batch LUFS on GPU** - Implement ITU-R BS.1770 in PyTorch for ~10× LUFS speedup
   175→3. **Use NVIDIA DALI** - GPU-accelerated audio pipeline
   176→
   177→---
   178→
   179→## Conclusion
   180→
   181→The A100-80GB GPU is **significantly underutilized** for this workload. The primary bottleneck is CPU-bound LUFS computation (pyloudnorm), which consumes 45% of processing time.
   182→
   183→**Recommended action:** Switch to CPU-only instances (c6i.4xlarge or similar) and scale horizontally. This will reduce costs by 80-90% while maintaining similar throughput.
   184→
   185→For 500k videos:
   186→- **8 × c6i.4xlarge nodes:** ~6 days, ~$780 total
   187→- **1 × A100 node:** ~46 days, ~$3,850 total
   188→
   189→The CPU approach is **5× cheaper** and **8× faster** due to parallelization.
   190→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

# GPU Pipeline Performance Analysis for A100-80GB

## Executive Summary

**Current Performance:** ~26 seg/s, ~7.5 videos/min on A100-80GB with 16 vCPUs

**Key Finding:** The A100 GPU is heavily underutilized. The bottleneck is CPU-bound metrics computation (pyloudnorm LUFS), not GPU operations.

**Recommendation:** For cost optimization, consider using CPU-only instances (e.g., c6i.4xlarge) instead of expensive GPU instances. The GPU provides minimal benefit for this workload.

---

## Benchmark Results

### Per-Operation Timing (Single Segment)

| Operation | Time (ms) | % of Total |
|-----------|-----------|------------|
| FLAC Decode | 17.9 | 31.0% |
| **Metrics (LUFS+Spectral)** | **26.1** | **45.3%** |
| GPU Resample (×4 rates) | 0.5 | 0.9% |
| WAV Encode (×4 rates) | 13.1 | 22.8% |
| **Total** | **57.7** | 100% |

**The GPU is only used for 0.9% of the processing time.**

### Parallelization Effectiveness

| Workers | Metrics-Only | Full Pipeline (GPU) | Full Pipeline (CPU) |
|---------|-------------|---------------------|---------------------|
| 4 | 76.3 seg/s | 51.2 seg/s | 42.3 seg/s |
| 8 | 96.8 seg/s | 50.7 seg/s | 50.1 seg/s |
| 12 | 94.4 seg/s | 50.8 seg/s | 48.6 seg/s |
| 16 | 92.2 seg/s | 49.4 seg/s | 46.8 seg/s |

**Key Insight:** GPU resampling provides only ~2% improvement over CPU resampling. The GPU is bottlenecked by:
1. CPU↔GPU memory transfer overhead
2. Serialized CUDA stream execution
3. The small batch size (single segment at a time)

### Real Data Test (Hindi, 20 videos)

- Segments loaded: 4,142
- Segments/video: 207
- Throughput: **26.1 seg/s**
- Videos/min: **7.56**
- Dropped (too short): 87.7%
- Outputs generated: 7,393

---

## Bottleneck Analysis

### Primary Bottleneck: pyloudnorm LUFS (45% of time)

The `compute_audio_metrics()` function calls `pyloudnorm.Meter.integrated_loudness()`, which is a pure Python implementation of ITU-R BS.1770.

**Options to optimize:**
1. **Skip LUFS for filtering** - If LUFS is only used for quality scoring, consider a simpler RMS-based approximation
2. **Batch LUFS on GPU** - Implement ITU-R BS.1770 K-weighting filters in PyTorch
3. **Use a faster LUFS library** - e.g., `ffmpeg-normalize` or custom C extension

### Secondary Bottleneck: FLAC Decode (31% of time)

Using `soundfile` (libsndfile) for FLAC decoding. Already parallelized.

**Options:**
1. Store audio as WAV in cleaned bucket (eliminates decode entirely)
2. Use `av` library (PyAV) which may be faster for some formats

### Tertiary Bottleneck: WAV Encode (23% of time)

Already parallelized. Limited optimization potential.

---

## Optimal Configuration

### For A100-80GB + 16 vCPUs

```bash
python prepare_shards.py hindi --gpu \
    --gpu-batch-size 128 \
    --gpu-decode-workers 12 \
    --gpu-encode-workers 12 \
    --output-format wav
```

| Parameter | Value | Rationale |
|-----------|-------|-----------|
| gpu-batch-size | 128 | Marginal impact; larger batches don't help |
| gpu-decode-workers | 12 | Optimal for 16 vCPUs (leaves headroom) |
| gpu-encode-workers | 12 | Same as decode |

### Expected Throughput

| Metric | Value |
|--------|-------|
| Segments/sec | 26-30 |
| Videos/min | 7.5-8.5 |
| 16k outputs/min | ~1,500 |

---

## Scaling Projections

### Single A100 Node

| Dataset Size | Time | Days |
|-------------|------|------|
| 10,000 videos | 22 hours | 0.9 |
| 50,000 videos | 110 hours | 4.6 |
| 100,000 videos | 220 hours | 9.2 |
| 500,000 videos | 1,100 hours | 45.8 |

### Horizontal Scaling (500k videos)

| Nodes | Time | Days | Cost Efficiency |
|-------|------|------|-----------------|
| 1 | 1,100h | 46 days | Baseline |
| 2 | 550h | 23 days | 2× |
| 4 | 275h | 11 days | 4× |
| 8 | 138h | 5.7 days | 8× |
| 16 | 69h | 2.9 days | 16× |
| 32 | 34h | 1.4 days | 32× |

**Scaling is linear** - no bottleneck on shared resources (each node processes independent videos).

---

## Cost Optimization Recommendations

### Option 1: Keep A100 (Current)

- **Cost:** ~$3.50/hour (p4d.24xlarge spot) or ~$10/hour (on-demand)
- **Throughput:** 7.5 videos/min
- **GPU utilization:** <5%
- **Verdict:** Inefficient; paying for unused GPU capacity

### Option 2: CPU-Only Instances (Recommended)

Replace A100 with CPU instances:

| Instance | vCPUs | Cost/hr | Est. Throughput | $/video |
|----------|-------|---------|-----------------|---------|
| c6i.4xlarge | 16 | $0.68 | 6.5 vid/min | $0.0017 |
| c6i.8xlarge | 32 | $1.36 | 10 vid/min | $0.0023 |
| c7i.4xlarge | 16 | $0.71 | 6.5 vid/min | $0.0018 |

**Savings:** 80-90% cost reduction vs A100 with similar throughput.

### Option 3: Hybrid Approach

Use A100 nodes only if future optimizations (batch GPU LUFS, batch GPU metrics) are implemented.

---

## Implementation Recommendations

### Immediate (No Code Changes)

1. **Use CPU instances** instead of A100 for cost efficiency
2. **Scale horizontally** with 4-8 CPU nodes for faster completion
3. **Use optimal config:** 12 decode workers, 12 encode workers, batch_size 128

### Short-term (Code Changes)

1. **Parallelize segment processing** - Current code processes segments sequentially after decode; fully parallelize the entire chain
2. **Remove GPU path** - It provides minimal benefit and adds complexity

### Long-term (Significant Changes)

1. **Store cleaned audio as WAV** - Eliminates FLAC decode overhead (31% savings)
2. **Batch LUFS on GPU** - Implement ITU-R BS.1770 in PyTorch for ~10× LUFS speedup
3. **Use NVIDIA DALI** - GPU-accelerated audio pipeline

---

## Conclusion

The A100-80GB GPU is **significantly underutilized** for this workload. The primary bottleneck is CPU-bound LUFS computation (pyloudnorm), which consumes 45% of processing time.

**Recommended action:** Switch to CPU-only instances (c6i.4xlarge or similar) and scale horizontally. This will reduce costs by 80-90% while maintaining similar throughput.

For 500k videos:
- **8 × c6i.4xlarge nodes:** ~6 days, ~$780 total
- **1 × A100 node:** ~46 days, ~$3,850 total

The CPU approach is **5× cheaper** and **8× faster** due to parallelization.


     1→# Audio Pipeline Optimization Strategy
     2→
     3→## Target: 500k Videos in <24 Hours
     4→
     5→### Current State
     6→```
     7→Performance:     26.1 seg/s, 7.56 videos/min
     8→Bottleneck:      pyloudnorm LUFS computation (45% of time)
     9→GPU Utilization: <5% (only resampling)
    10→Time for 500k:   ~46 days
    11→```
    12→
    13→### Target State
    14→```
    15→Required:        500k videos / 24h = 347 videos/min
    16→Speedup needed:  347 / 7.56 = 46x
    17→```
    18→
    19→---
    20→
    21→## Strategy Overview
    22→
    23→We'll achieve this through **3 parallel optimization tracks**:
    24→
    25→| Track | Optimization | Expected Speedup | Cost Impact |
    26→|-------|--------------|------------------|-------------|
    27→| **A** | GPU-accelerated LUFS + metrics | 4-6x | Same hardware |
    28→| **B** | Pipeline parallelization | 2-3x | Same hardware |
    29→| **C** | Horizontal scaling | 8-16x | Linear scaling |
    30→
    31→**Combined target: 48-288x speedup → 500k in 4-24 hours**
    32→
    33→---
    34→
    35→## Track A: GPU-Accelerated Audio Metrics
    36→
    37→### The Problem with pyloudnorm
    38→
    39→pyloudnorm is pure Python and computes LUFS sequentially:
    40→1. K-weighting filter (2x IIR biquad cascades)
    41→2. Mean square computation
    42→3. Gating (400ms blocks with 75% overlap)
    43→4. Integration
    44→
    45→**Solution: GPU-native LUFS with torchaudio**
    46→
    47→```python
    48→# Current: 26.1ms per segment (sequential, CPU)
    49→# Target:  0.5ms per segment (batched, GPU)
    50→```
    51→
    52→### GPU LUFS Implementation
    53→
    54→The EBU R128 LUFS algorithm can be fully parallelized:
    55→
    56→1. **K-weighting filters** → `torchaudio.functional.biquad` (batched)
    57→2. **Block mean square** → `torch.unfold` + batched mean
    58→3. **Gating** → Vectorized threshold comparison
    59→4. **Integration** → Batched reduction
    60→
    61→### Performance Projection
    62→
    63→| Operation | CPU (pyloudnorm) | GPU (batched 128) | Speedup |
    64→|-----------|------------------|-------------------|---------|
    65→| LUFS | 26.1ms | ~0.8ms | 32x |
    66→| Spectral | 8ms | ~0.3ms | 26x |
    67→| Resample | 12ms | 0.5ms | 24x |
    68→| **Total** | 57.6ms | ~5ms | **11x** |
    69→
    70→---
    71→
    72→## Track B: Pipeline Architecture Optimization
    73→
    74→### Current Architecture (Sequential)
    75→```
    76→Download → Decode → Metrics → Chunk → Resample → Encode → Write
    77→           [------------ Sequential per segment ------------]
    78→```
    79→
    80→### Optimized Architecture (Pipelined + Batched)
    81→```
    82→┌─────────────────────────────────────────────────────────────┐
    83→│                    PRODUCER PROCESSES                        │
    84→│  Download Pool (16 workers) → Decode Pool (8 workers)       │
    85→│            ↓                         ↓                       │
    86→│        [Download Queue]         [Segment Queue]              │
    87→└─────────────────────────────────────────────────────────────┘
    88→                              ↓
    89→┌─────────────────────────────────────────────────────────────┐
    90→│                    GPU BATCH PROCESSOR                       │
    91→│  Batch Accumulator (128 segments) → GPU Kernel              │
    92→│  - K-weight filter (batched)                                 │
    93→│  - LUFS computation (batched)                                │
    94→│  - Spectral analysis (batched FFT)                           │
    95→│  - Resampling (batched)                                      │
    96→└─────────────────────────────────────────────────────────────┘
    97→                              ↓
    98→┌─────────────────────────────────────────────────────────────┐
    99→│                   CONSUMER PROCESSES                         │
   100→│  Encode Pool (12 workers) → Shard Writers (4 workers)       │
   101→│            ↓                         ↓                       │
   102→│       [Output Queue]           [Upload Queue]                │
   103→└─────────────────────────────────────────────────────────────┘
   104→```
   105→
   106→### Key Optimizations
   107→
   108→1. **Async I/O for downloads** - Don't block GPU on network
   109→2. **Pre-fetch next batch** - GPU never waits for data
   110→3. **Streaming encode** - Encode while GPU processes next batch
   111→4. **Batched shard writes** - Reduce filesystem syscalls
   112→
   113→---
   114→
   115→## Track C: Horizontal Scaling Strategy
   116→
   117→### Compute Requirements
   118→
   119→For 500k videos in 24 hours with optimized code (11x speedup):
   120→
   121→```
   122→Single node: 7.56 × 11 = 83 videos/min
   123→Required:    347 videos/min
   124→Nodes needed: ceil(347 / 83) = 5 nodes
   125→Safety margin: 8 nodes (2.3x buffer)
   126→```
   127→
   128→### Recommended Instance Types
   129→
   130→| Provider | Instance | vCPUs | GPU | Memory | Cost/hr |
   131→|----------|----------|-------|-----|--------|---------|
   132→| AWS | g5.4xlarge | 16 | A10G 24GB | 64GB | ~$1.00 |
   133→| AWS | g5.8xlarge | 32 | A10G 24GB | 128GB | ~$2.00 |
   134→| AWS | p4d.24xlarge | 96 | 8×A100 | 1.5TB | ~$32.77 |
   135→| GCP | g2-standard-16 | 16 | L4 24GB | 64GB | ~$0.80 |
   136→
   137→**Recommendation: 8× g5.4xlarge ($8/hr total) or 2× g5.8xlarge with better CPU/GPU ratio**
   138→
   139→### Partitioning Strategy
   140→
   141→```sql
   142→-- Each worker gets deterministic partition
   143→SELECT youtube_id, r2_tar_key
   144→FROM videos
   145→WHERE language = $1
   146→  AND status = 'COMPLETED'
   147→  AND mod(abs(hashtext(youtube_id)), $partition_count) = $partition_index
   148→```
   149→
   150→This ensures:
   151→- No overlap between workers
   152→- Deterministic assignment (restart-safe)
   153→- Even distribution
   154→
   155→---
   156→
   157→## Monitoring & Orchestration
   158→
   159→### Progress Tracking Schema (Supabase)
   160→
   161→```sql
   162→CREATE TABLE pipeline_runs (
   163→  run_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
   164→  language TEXT NOT NULL,
   165→  total_videos INT NOT NULL,
   166→  started_at TIMESTAMPTZ DEFAULT NOW(),
   167→  completed_at TIMESTAMPTZ,
   168→  status TEXT DEFAULT 'running'
   169→);
   170→
   171→CREATE TABLE worker_progress (
   172→  worker_id TEXT PRIMARY KEY,
   173→  run_id UUID REFERENCES pipeline_runs(run_id),
   174→  partition_index INT NOT NULL,
   175→  partition_count INT NOT NULL,
   176→  hostname TEXT,
   177→  instance_type TEXT,
   178→
   179→  -- Progress counters
   180→  videos_total INT DEFAULT 0,
   181→  videos_processed INT DEFAULT 0,
   182→  segments_total INT DEFAULT 0,
   183→  segments_processed INT DEFAULT 0,
   184→  segments_dropped INT DEFAULT 0,
   185→
   186→  -- Performance metrics
   187→  videos_per_minute FLOAT DEFAULT 0,
   188→  segments_per_second FLOAT DEFAULT 0,
   189→  gpu_utilization FLOAT DEFAULT 0,
   190→
   191→  -- Status
   192→  status TEXT DEFAULT 'starting',
   193→  last_heartbeat TIMESTAMPTZ DEFAULT NOW(),
   194→  last_error TEXT,
   195→  error_count INT DEFAULT 0,
   196→
   197→  started_at TIMESTAMPTZ DEFAULT NOW(),
   198→  updated_at TIMESTAMPTZ DEFAULT NOW()
   199→);
   200→
   201→CREATE TABLE processing_errors (
   202→  id SERIAL PRIMARY KEY,
   203→  worker_id TEXT,
   204→  run_id UUID,
   205→  video_id TEXT,
   206→  error_type TEXT,
   207→  error_message TEXT,
   208→  stack_trace TEXT,
   209→  created_at TIMESTAMPTZ DEFAULT NOW()
   210→);
   211→
   212→-- Real-time aggregation view
   213→CREATE VIEW run_progress AS
   214→SELECT
   215→  r.run_id,
   216→  r.language,
   217→  r.total_videos,
   218→  SUM(w.videos_processed) as processed,
   219→  SUM(w.videos_total) as assigned,
   220→  AVG(w.videos_per_minute) as avg_rate,
   221→  SUM(w.videos_per_minute) as total_rate,
   222→  COUNT(*) FILTER (WHERE w.status = 'running') as active_workers,
   223→  COUNT(*) FILTER (WHERE w.status = 'failed') as failed_workers,
   224→  EXTRACT(EPOCH FROM (NOW() - r.started_at)) / 3600 as hours_elapsed,
   225→  (r.total_videos - SUM(w.videos_processed)) / NULLIF(SUM(w.videos_per_minute), 0) / 60 as hours_remaining
   226→FROM pipeline_runs r
   227→LEFT JOIN worker_progress w ON w.run_id = r.run_id
   228→GROUP BY r.run_id;
   229→```
   230→
   231→### Worker Heartbeat & Monitoring
   232→
   233→```python
   234→class ProgressReporter:
   235→    def __init__(self, supabase_url: str, supabase_key: str, run_id: str, worker_id: str):
   236→        self.client = create_client(supabase_url, supabase_key)
   237→        self.run_id = run_id
   238→        self.worker_id = worker_id
   239→        self.last_report = time.time()
   240→        self.report_interval = 10  # seconds
   241→
   242→    def update(self, videos_processed: int, segments_processed: int,
   243→               videos_per_minute: float, segments_per_second: float):
   244→        now = time.time()
   245→        if now - self.last_report < self.report_interval:
   246→            return
   247→
   248→        self.client.table('worker_progress').upsert({
   249→            'worker_id': self.worker_id,
   250→            'run_id': self.run_id,
   251→            'videos_processed': videos_processed,
   252→            'segments_processed': segments_processed,
   253→            'videos_per_minute': videos_per_minute,
   254→            'segments_per_second': segments_per_second,
   255→            'last_heartbeat': datetime.utcnow().isoformat(),
   256→            'status': 'running'
   257→        }).execute()
   258→        self.last_report = now
   259→```
   260→
   261→---
   262→
   263→## Implementation Phases
   264→
   265→### Phase 1: GPU-Accelerated Metrics (Day 1)
   266→- [ ] Implement GPU LUFS computation
   267→- [ ] Implement GPU spectral analysis
   268→- [ ] Batch processing with async I/O
   269→- [ ] Benchmark: Target 80+ seg/s single GPU
   270→
   271→### Phase 2: Horizontal Scaling Infrastructure (Day 1)
   272→- [ ] Supabase schema setup
   273→- [ ] Worker orchestration script
   274→- [ ] Progress monitoring dashboard
   275→- [ ] Auto-recovery on failure
   276→
   277→### Phase 3: Deployment & Execution (Day 2)
   278→- [ ] Deploy to 8 GPU instances
   279→- [ ] Start processing all 12 languages
   280→- [ ] Monitor and adjust
   281→- [ ] Handle failures and retries
   282→
   283→---
   284→
   285→## Cost Estimate (24-hour run)
   286→
   287→| Component | Units | Cost/hr | Hours | Total |
   288→|-----------|-------|---------|-------|-------|
   289→| g5.4xlarge | 8 | $1.00 | 24 | $192 |
   290→| R2 egress | ~5TB | - | - | ~$0 (free) |
   291→| Supabase | Pro | - | - | $25/mo |
   292→| **Total** | | | | **~$220** |
   293→
   294→---
   295→
   296→## Quick Start
   297→
   298→```bash
   299→# 1. Deploy infrastructure
   300→./deploy/setup_workers.sh 8  # Spin up 8 GPU instances
   301→
   302→# 2. Start orchestrator
   303→python orchestrator.py --language all --workers 8 --run-id $(uuidgen)
   304→
   305→# 3. Monitor progress
   306→python monitor.py --run-id <run_id>  # Real-time dashboard
   307→```
   308→
   309→---
   310→
   311→## Files to Create
   312→
   313→1. `gpu_audio_metrics.py` - GPU-accelerated LUFS and spectral analysis
   314→2. `optimized_pipeline.py` - Pipelined batch processor
   315→3. `orchestrator.py` - Multi-node job orchestration
   316→4. `monitor.py` - Real-time progress dashboard
   317→5. `deploy/` - Infrastructure as code

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

# Audio Pipeline Optimization Strategy

## Target: 500k Videos in <24 Hours

### Current State
```
Performance:     26.1 seg/s, 7.56 videos/min
Bottleneck:      pyloudnorm LUFS computation (45% of time)
GPU Utilization: <5% (only resampling)
Time for 500k:   ~46 days
```

### Target State
```
Required:        500k videos / 24h = 347 videos/min
Speedup needed:  347 / 7.56 = 46x
```

---

## Strategy Overview

We'll achieve this through **3 parallel optimization tracks**:

| Track | Optimization | Expected Speedup | Cost Impact |
|-------|--------------|------------------|-------------|
| **A** | GPU-accelerated LUFS + metrics | 4-6x | Same hardware |
| **B** | Pipeline parallelization | 2-3x | Same hardware |
| **C** | Horizontal scaling | 8-16x | Linear scaling |

**Combined target: 48-288x speedup → 500k in 4-24 hours**

---

## Track A: GPU-Accelerated Audio Metrics

### The Problem with pyloudnorm

pyloudnorm is pure Python and computes LUFS sequentially:
1. K-weighting filter (2x IIR biquad cascades)
2. Mean square computation
3. Gating (400ms blocks with 75% overlap)
4. Integration

**Solution: GPU-native LUFS with torchaudio**

```python
# Current: 26.1ms per segment (sequential, CPU)
# Target:  0.5ms per segment (batched, GPU)
```

### GPU LUFS Implementation

The EBU R128 LUFS algorithm can be fully parallelized:

1. **K-weighting filters** → `torchaudio.functional.biquad` (batched)
2. **Block mean square** → `torch.unfold` + batched mean
3. **Gating** → Vectorized threshold comparison
4. **Integration** → Batched reduction

### Performance Projection

| Operation | CPU (pyloudnorm) | GPU (batched 128) | Speedup |
|-----------|------------------|-------------------|---------|
| LUFS | 26.1ms | ~0.8ms | 32x |
| Spectral | 8ms | ~0.3ms | 26x |
| Resample | 12ms | 0.5ms | 24x |
| **Total** | 57.6ms | ~5ms | **11x** |

---

## Track B: Pipeline Architecture Optimization

### Current Architecture (Sequential)
```
Download → Decode → Metrics → Chunk → Resample → Encode → Write
           [------------ Sequential per segment ------------]
```

### Optimized Architecture (Pipelined + Batched)
```
┌─────────────────────────────────────────────────────────────┐
│                    PRODUCER PROCESSES                        │
│  Download Pool (16 workers) → Decode Pool (8 workers)       │
│            ↓                         ↓                       │
│        [Download Queue]         [Segment Queue]              │
└─────────────────────────────────────────────────────────────┘
                              ↓
┌─────────────────────────────────────────────────────────────┐
│                    GPU BATCH PROCESSOR                       │
│  Batch Accumulator (128 segments) → GPU Kernel              │
│  - K-weight filter (batched)                                 │
│  - LUFS computation (batched)                                │
│  - Spectral analysis (batched FFT)                           │
│  - Resampling (batched)                                      │
└─────────────────────────────────────────────────────────────┘
                              ↓
┌─────────────────────────────────────────────────────────────┐
│                   CONSUMER PROCESSES                         │
│  Encode Pool (12 workers) → Shard Writers (4 workers)       │
│            ↓                         ↓                       │
│       [Output Queue]           [Upload Queue]                │
└─────────────────────────────────────────────────────────────┘
```

### Key Optimizations

1. **Async I/O for downloads** - Don't block GPU on network
2. **Pre-fetch next batch** - GPU never waits for data
3. **Streaming encode** - Encode while GPU processes next batch
4. **Batched shard writes** - Reduce filesystem syscalls

---

## Track C: Horizontal Scaling Strategy

### Compute Requirements

For 500k videos in 24 hours with optimized code (11x speedup):

```
Single node: 7.56 × 11 = 83 videos/min
Required:    347 videos/min
Nodes needed: ceil(347 / 83) = 5 nodes
Safety margin: 8 nodes (2.3x buffer)
```

### Recommended Instance Types

| Provider | Instance | vCPUs | GPU | Memory | Cost/hr |
|----------|----------|-------|-----|--------|---------|
| AWS | g5.4xlarge | 16 | A10G 24GB | 64GB | ~$1.00 |
| AWS | g5.8xlarge | 32 | A10G 24GB | 128GB | ~$2.00 |
| AWS | p4d.24xlarge | 96 | 8×A100 | 1.5TB | ~$32.77 |
| GCP | g2-standard-16 | 16 | L4 24GB | 64GB | ~$0.80 |

**Recommendation: 8× g5.4xlarge ($8/hr total) or 2× g5.8xlarge with better CPU/GPU ratio**

### Partitioning Strategy

```sql
-- Each worker gets deterministic partition
SELECT youtube_id, r2_tar_key
FROM videos
WHERE language = $1
  AND status = 'COMPLETED'
  AND mod(abs(hashtext(youtube_id)), $partition_count) = $partition_index
```

This ensures:
- No overlap between workers
- Deterministic assignment (restart-safe)
- Even distribution

---

## Monitoring & Orchestration

### Progress Tracking Schema (Supabase)

```sql
CREATE TABLE pipeline_runs (
  run_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  language TEXT NOT NULL,
  total_videos INT NOT NULL,
  started_at TIMESTAMPTZ DEFAULT NOW(),
  completed_at TIMESTAMPTZ,
  status TEXT DEFAULT 'running'
);

CREATE TABLE worker_progress (
  worker_id TEXT PRIMARY KEY,
  run_id UUID REFERENCES pipeline_runs(run_id),
  partition_index INT NOT NULL,
  partition_count INT NOT NULL,
  hostname TEXT,
  instance_type TEXT,

  -- Progress counters
  videos_total INT DEFAULT 0,
  videos_processed INT DEFAULT 0,
  segments_total INT DEFAULT 0,
  segments_processed INT DEFAULT 0,
  segments_dropped INT DEFAULT 0,

  -- Performance metrics
  videos_per_minute FLOAT DEFAULT 0,
  segments_per_second FLOAT DEFAULT 0,
  gpu_utilization FLOAT DEFAULT 0,

  -- Status
  status TEXT DEFAULT 'starting',
  last_heartbeat TIMESTAMPTZ DEFAULT NOW(),
  last_error TEXT,
  error_count INT DEFAULT 0,

  started_at TIMESTAMPTZ DEFAULT NOW(),
  updated_at TIMESTAMPTZ DEFAULT NOW()
);

CREATE TABLE processing_errors (
  id SERIAL PRIMARY KEY,
  worker_id TEXT,
  run_id UUID,
  video_id TEXT,
  error_type TEXT,
  error_message TEXT,
  stack_trace TEXT,
  created_at TIMESTAMPTZ DEFAULT NOW()
);

-- Real-time aggregation view
CREATE VIEW run_progress AS
SELECT
  r.run_id,
  r.language,
  r.total_videos,
  SUM(w.videos_processed) as processed,
  SUM(w.videos_total) as assigned,
  AVG(w.videos_per_minute) as avg_rate,
  SUM(w.videos_per_minute) as total_rate,
  COUNT(*) FILTER (WHERE w.status = 'running') as active_workers,
  COUNT(*) FILTER (WHERE w.status = 'failed') as failed_workers,
  EXTRACT(EPOCH FROM (NOW() - r.started_at)) / 3600 as hours_elapsed,
  (r.total_videos - SUM(w.videos_processed)) / NULLIF(SUM(w.videos_per_minute), 0) / 60 as hours_remaining
FROM pipeline_runs r
LEFT JOIN worker_progress w ON w.run_id = r.run_id
GROUP BY r.run_id;
```

### Worker Heartbeat & Monitoring

```python
class ProgressReporter:
    def __init__(self, supabase_url: str, supabase_key: str, run_id: str, worker_id: str):
        self.client = create_client(supabase_url, supabase_key)
        self.run_id = run_id
        self.worker_id = worker_id
        self.last_report = time.time()
        self.report_interval = 10  # seconds

    def update(self, videos_processed: int, segments_processed: int,
               videos_per_minute: float, segments_per_second: float):
        now = time.time()
        if now - self.last_report < self.report_interval:
            return

        self.client.table('worker_progress').upsert({
            'worker_id': self.worker_id,
            'run_id': self.run_id,
            'videos_processed': videos_processed,
            'segments_processed': segments_processed,
            'videos_per_minute': videos_per_minute,
            'segments_per_second': segments_per_second,
            'last_heartbeat': datetime.utcnow().isoformat(),
            'status': 'running'
        }).execute()
        self.last_report = now
```

---

## Implementation Phases

### Phase 1: GPU-Accelerated Metrics (Day 1)
- [ ] Implement GPU LUFS computation
- [ ] Implement GPU spectral analysis
- [ ] Batch processing with async I/O
- [ ] Benchmark: Target 80+ seg/s single GPU

### Phase 2: Horizontal Scaling Infrastructure (Day 1)
- [ ] Supabase schema setup
- [ ] Worker orchestration script
- [ ] Progress monitoring dashboard
- [ ] Auto-recovery on failure

### Phase 3: Deployment & Execution (Day 2)
- [ ] Deploy to 8 GPU instances
- [ ] Start processing all 12 languages
- [ ] Monitor and adjust
- [ ] Handle failures and retries

---

## Cost Estimate (24-hour run)

| Component | Units | Cost/hr | Hours | Total |
|-----------|-------|---------|-------|-------|
| g5.4xlarge | 8 | $1.00 | 24 | $192 |
| R2 egress | ~5TB | - | - | ~$0 (free) |
| Supabase | Pro | - | - | $25/mo |
| **Total** | | | | **~$220** |

---

## Quick Start

```bash
# 1. Deploy infrastructure
./deploy/setup_workers.sh 8  # Spin up 8 GPU instances

# 2. Start orchestrator
python orchestrator.py --language all --workers 8 --run-id $(uuidgen)

# 3. Monitor progress
python monitor.py --run-id <run_id>  # Real-time dashboard
```

---

## Files to Create

1. `gpu_audio_metrics.py` - GPU-accelerated LUFS and spectral analysis
2. `optimized_pipeline.py` - Pipelined batch processor
3. `orchestrator.py` - Multi-node job orchestration
4. `monitor.py` - Real-time progress dashboard
5. `deploy/` - Infrastructure as code


Now let me check the existing documentation and understand the X-Codec structure:


Bash

ls -la /home/ubuntu/maya3_prepare/X-Codec-2.0/ | head -20


total 120
drwxr-xr-x 9 ubuntu ubuntu  4096 Jan 19 05:21 .
drwxrwxr-x 8 ubuntu ubuntu  4096 Jan 19 21:45 ..
drwxr-xr-x 3 ubuntu ubuntu  4096 Jan 19 04:03 common
drwxr-xr-x 6 ubuntu ubuntu  4096 Jan 19 04:03 config
drwxr-xr-x 2 ubuntu ubuntu  4096 Jan 19 04:03 criterions
-rw-rw-r-- 1 ubuntu ubuntu  4267 Jan 19 05:21 data_module.py
-rw-r--r-- 1 ubuntu ubuntu  2315 Jan 19 04:03 get_tsv.py
drwxr-xr-x 8 ubuntu ubuntu  4096 Jan 19 22:37 .git
-rw-r--r-- 1 ubuntu ubuntu  5884 Jan 19 04:03 inference.py
-rw-r--r-- 1 ubuntu ubuntu  8256 Jan 19 04:03 inference_save_code.py
-rw-r--r-- 1 ubuntu ubuntu  1130 Jan 19 04:03 large_scale_save_code.sh
-rw-r--r-- 1 ubuntu ubuntu  1064 Jan 19 04:03 LICENSE
-rw-r--r-- 1 ubuntu ubuntu 15869 Jan 19 04:03 lightning_module.py
drwxr-xr-x 2 ubuntu ubuntu  4096 Jan 19 04:03 module
-rw-r--r-- 1 ubuntu ubuntu  1038 Jan 19 04:03 preprocess.py
-rw-r--r-- 1 ubuntu ubuntu  3099 Jan 19 04:03 README.md
-rw-r--r-- 1 ubuntu ubuntu  2430 Jan 19 04:03 requirements.txt
drwxr-xr-x 4 ubuntu ubuntu  4096 Jan 19 04:03 test_audio
-rw-r--r-- 1 ubuntu ubuntu  2421 Jan 19 04:03 train.py


     1→import os
     2→
     3→import re
     4→import torch
     5→import torch.nn as nn
     6→import torch.nn.functional as F
     7→import torchaudio
     8→import numpy as np
     9→import pytorch_lightning as pl
    10→import random
    11→import librosa
    12→from os.path import basename, exists, join
    13→from torch.utils.data import Dataset, DataLoader
    14→import hydra
    15→import utils
    16→import torchaudio
    17→from transformers import AutoFeatureExtractor
    18→from torchaudio.transforms import Resample
    19→from tqdm import tqdm
    20→from torchaudio.transforms import Resample
    21→class DataModule(pl.LightningDataModule):
    22→    def __init__(self, cfg):
    23→        super().__init__()
    24→        self.cfg = cfg
    25→
    26→        ocwd = hydra.utils.get_original_cwd()
    27→        self.ocwd = ocwd
    28→
    29→    def get_loader(self, phase):
    30→        phase_cfg = self.cfg.dataset.get(phase)
    31→        batch_size = phase_cfg.batch_size
    32→        ds = FSDataset(phase, self.cfg)
    33→        # ds = FSDataset_add_STFT(phase, self.cfg)
    34→        dl = DataLoader(ds,
    35→                        batch_size=batch_size,
    36→                        shuffle=phase_cfg.shuffle,
    37→                        num_workers=28,
    38→                        collate_fn=ds.collate_fn,
    39→                        pin_memory=True,
    40→                        persistent_workers=True)
    41→
    42→        return dl
    43→
    44→    def train_dataloader(self):
    45→        return self.get_loader('train')
    46→
    47→    def val_dataloader(self):
    48→        return self.get_loader('val')
    49→
    50→    def test_dataloader(self):
    51→        pass
    52→
    53→class FSDataset(Dataset):
    54→    """Dataset batching wav, mel
    55→    and other acoustic features
    56→
    57→    Args:
    58→        phase: train, val, test
    59→        cfg: hydra config
    60→    """
    61→    def __init__(self, phase, cfg):
    62→        self.phase = phase
    63→        self.cfg = cfg
    64→        self.phase_cfg = cfg.dataset.get(phase)
    65→        self.ocwd = hydra.utils.get_original_cwd()
    66→
    67→        self.sr = cfg.preprocess.audio.sr
    68→
    69→        # self.filelist = utils.read_filelist(join(self.ocwd, self.phase_cfg.filelist))
    70→        self.filelist = self.get_filelist(self.phase_cfg.filelist)
    71→        self.min_audio_length = cfg.dataset.min_audio_length
    72→        self.feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/w2v-bert-2.0")
    73→    def __len__(self):
    74→        return len(self.filelist)
    75→
    76→    def load_wav(self, path):
    77→        wav, sr = librosa.load(path, sr=self.sr)
    78→        return wav
    79→
    80→    def get_filelist(self, fpath):
    81→        with open(fpath, 'r') as f:
    82→            # flist = [l.strip() for l in f if l.strip()]
    83→            flist = [l.strip().split('\t')[0] for l in f if l.strip()]
    84→        return flist
    85→
    86→    def __getitem__(self, idx):
    87→        # (  wavpath,fid) = self.filelist[idx]
    88→        wavpath  = self.filelist[idx]
    89→        wavpath_full = join(self.cfg.preprocess.datasets.LibriSpeech.root, wavpath)
    90→        # wav = self.load_wav(wavpath)
    91→        # wav = torch.from_numpy(wav)
    92→
    93→        wav,sr=torchaudio.load(wavpath_full)
    94→
    95→
    96→        if sr != 16000:
    97→            wav = Resample(sr, 16000)(wav)
    98→        wav = wav[0,:]
    99→        length = wav.shape[0]
   100→        # length = wav.shape[1]
   101→        if length < self.min_audio_length:
   102→            wav = F.pad(wav, (0, self.min_audio_length - length))
   103→            length = wav.shape[0]
   104→        i = random.randint(0, length-self.min_audio_length)
   105→        wav = wav[i:i+self.min_audio_length]
   106→
   107→        wav_pad = F.pad(wav, (160, 160))
   108→        feat = self.feature_extractor(wav_pad, sampling_rate=16000, return_tensors="pt") .data['input_features']
   109→        out = {
   110→
   111→            'wav': wav,
   112→            'feat': feat,
   113→            # 'paths': wavpath_full
   114→        }
   115→
   116→        return out
   117→
   118→    def collate_fn(self, bs):
   119→
   120→        wavs = [b['wav'] for b in bs]
   121→        wavs = torch.stack(wavs)
   122→        feats = [b['feat'] for b in bs]
   123→        feats = torch.stack(feats)
   124→        out = {
   125→
   126→            'wav': wavs,
   127→            'feats': feats,
   128→            # 'paths': [b['paths'] for b in bs]
   129→        }
   130→        return out
   131→
   132→@hydra.main(config_path='config', config_name='default', version_base=None)
   133→def main(cfg):
   134→
   135→    data_module = DataModule(cfg)
   136→
   137→
   138→    train_loader = data_module.val_dataloader()
   139→
   140→
   141→    valid_filelist = []
   142→
   143→
   144→    for batch_idx, batch in enumerate(tqdm(train_loader, desc="Processing batches", unit="batch")):
   145→
   146→        wavs = batch['wav']
   147→
   148→
   149→if __name__ == "__main__":
   150→    main()
   151→
   152→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

import os

import re
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchaudio
import numpy as np
import pytorch_lightning as pl
import random
import librosa
from os.path import basename, exists, join
from torch.utils.data import Dataset, DataLoader
import hydra
import utils
import torchaudio
from transformers import AutoFeatureExtractor
from torchaudio.transforms import Resample
from tqdm import tqdm
from torchaudio.transforms import Resample
class DataModule(pl.LightningDataModule):
    def __init__(self, cfg):
        super().__init__()
        self.cfg = cfg

        ocwd = hydra.utils.get_original_cwd()
        self.ocwd = ocwd

    def get_loader(self, phase):
        phase_cfg = self.cfg.dataset.get(phase)
        batch_size = phase_cfg.batch_size
        ds = FSDataset(phase, self.cfg)
        # ds = FSDataset_add_STFT(phase, self.cfg)
        dl = DataLoader(ds,
                        batch_size=batch_size,
                        shuffle=phase_cfg.shuffle,
                        num_workers=28,
                        collate_fn=ds.collate_fn,
                        pin_memory=True,
                        persistent_workers=True)

        return dl

    def train_dataloader(self):
        return self.get_loader('train')

    def val_dataloader(self):
        return self.get_loader('val')

    def test_dataloader(self):
        pass

class FSDataset(Dataset):
    """Dataset batching wav, mel
    and other acoustic features

    Args:
        phase: train, val, test
        cfg: hydra config
    """
    def __init__(self, phase, cfg):
        self.phase = phase
        self.cfg = cfg
        self.phase_cfg = cfg.dataset.get(phase)
        self.ocwd = hydra.utils.get_original_cwd()

        self.sr = cfg.preprocess.audio.sr

        # self.filelist = utils.read_filelist(join(self.ocwd, self.phase_cfg.filelist))
        self.filelist = self.get_filelist(self.phase_cfg.filelist)
        self.min_audio_length = cfg.dataset.min_audio_length
        self.feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/w2v-bert-2.0")
    def __len__(self):
        return len(self.filelist)

    def load_wav(self, path):
        wav, sr = librosa.load(path, sr=self.sr)
        return wav

    def get_filelist(self, fpath):
        with open(fpath, 'r') as f:
            # flist = [l.strip() for l in f if l.strip()]
            flist = [l.strip().split('\t')[0] for l in f if l.strip()]
        return flist

    def __getitem__(self, idx):
        # (  wavpath,fid) = self.filelist[idx]
        wavpath  = self.filelist[idx]
        wavpath_full = join(self.cfg.preprocess.datasets.LibriSpeech.root, wavpath)
        # wav = self.load_wav(wavpath)
        # wav = torch.from_numpy(wav)

        wav,sr=torchaudio.load(wavpath_full)

        if sr != 16000:
            wav = Resample(sr, 16000)(wav)
        wav = wav[0,:]
        length = wav.shape[0]
        # length = wav.shape[1]
        if length < self.min_audio_length:
            wav = F.pad(wav, (0, self.min_audio_length - length))
            length = wav.shape[0]
        i = random.randint(0, length-self.min_audio_length)
        wav = wav[i:i+self.min_audio_length]

        wav_pad = F.pad(wav, (160, 160))
        feat = self.feature_extractor(wav_pad, sampling_rate=16000, return_tensors="pt") .data['input_features']
        out = {

            'wav': wav,
            'feat': feat,
            # 'paths': wavpath_full
        }

        return out

    def collate_fn(self, bs):

        wavs = [b['wav'] for b in bs]
        wavs = torch.stack(wavs)
        feats = [b['feat'] for b in bs]
        feats = torch.stack(feats)
        out = {

            'wav': wavs,
            'feats': feats,
            # 'paths': [b['paths'] for b in bs]
        }
        return out

@hydra.main(config_path='config', config_name='default', version_base=None)
def main(cfg):

    data_module = DataModule(cfg)

    train_loader = data_module.val_dataloader()

    valid_filelist = []

    for batch_idx, batch in enumerate(tqdm(train_loader, desc="Processing batches", unit="batch")):

        wavs = batch['wav']

if __name__ == "__main__":
    main()


     1→import os
     2→
     3→import random
     4→import hydra
     5→import numpy as np
     6→import librosa
     7→import torch
     8→import torch.nn as nn
     9→import torch.nn.functional as F
    10→import torch.optim as optim
    11→import pytorch_lightning as pl
    12→from vq import CodecEncoder,  CodecDecoderVocos
    13→from module import HiFiGANMultiPeriodDiscriminator, SpecDiscriminator
    14→from criterions import GANLoss, MultiResolutionMelSpectrogramLoss, MultiResolutionSTFTLoss
    15→from common.schedulers import WarmupLR
    16→from transformers import AutoModel
    17→from vq.module import SemanticDecoder,SemanticEncoder
    18→from transformers import AutoFeatureExtractor, Wav2Vec2BertModel
    19→import sys
    20→sys.path.append('./eval_tools/tools/speaker_verification')    # We use wavlm_large_finetune as a vadidation metric during training, https://github.com/microsoft/UniSpeech/tree/main/downstreams/speaker_verification
    21→from  verification import init_model
    22→model_spk = init_model('wavlm_large','/aifs4su/data/zheny/models_fd_ckpt/wavlm_large_finetune.pth')
    23→
    24→
    25→
    26→class CodecLightningModule(pl.LightningModule):
    27→    def __init__(self, cfg):
    28→        super().__init__()
    29→        self.cfg = cfg
    30→        self.ocwd = hydra.utils.get_original_cwd()
    31→        self.construct_model()
    32→        self.construct_criteria()
    33→        self.save_hyperparameters()
    34→        self.automatic_optimization = False
    35→
    36→    def construct_model(self):
    37→        # 初始化 Codec Encoder
    38→
    39→        enccfg = self.cfg.model.codec_encoder
    40→
    41→
    42→        self.CodecEnc = CodecEncoder(
    43→
    44→            ngf=enccfg.ngf,
    45→            up_ratios=enccfg.up_ratios,
    46→            dilations=enccfg.dilations,
    47→            hidden_dim=enccfg['hidden_dim'],
    48→            depth=enccfg['depth'],
    49→            heads=enccfg['heads'],
    50→            pos_meb_dim=enccfg['pos_meb_dim'],
    51→        )
    52→
    53→        # 初始化 Codec Decoder
    54→        deccfg = self.cfg.model.codec_decoder
    55→
    56→        self.generator = CodecDecoderVocos(
    57→            hidden_dim=deccfg.hidden_dim,
    58→            depth=deccfg.depth,
    59→            heads=deccfg.heads,
    60→            pos_meb_dim=deccfg.pos_meb_dim,
    61→            hop_length=320,
    62→            vq_num_quantizers=deccfg.vq_num_quantizers,  # VQ 量化器数量
    63→            vq_dim=deccfg.vq_dim,                   # VQ 维度
    64→            vq_commit_weight=deccfg.vq_commit_weight,    # VQ 提交权重
    65→            vq_weight_init=deccfg.vq_weight_init,         # VQ 权重初始化
    66→            vq_full_commit_loss=deccfg.vq_full_commit_loss,  # 是否使用完整的提交损失
    67→            codebook_size=deccfg.codebook_size,            # 码本大小
    68→            codebook_dim=deccfg.codebook_dim ,              # 码本维度
    69→                  # 隐藏层维度
    70→        )
    71→
    72→
    73→
    74→        # 初始化 MultiPeriod Discriminator
    75→        mpdcfg = self.cfg.model.mpd
    76→        self.discriminator = HiFiGANMultiPeriodDiscriminator(
    77→            periods=mpdcfg.periods,
    78→            max_downsample_channels=mpdcfg.max_downsample_channels,
    79→            channels=mpdcfg.channels,
    80→            channel_increasing_factor=mpdcfg.channel_increasing_factor,
    81→        )
    82→
    83→        # 初始化 Spectral Discriminator
    84→        mstftcfg = self.cfg.model.mstft
    85→        self.spec_discriminator = SpecDiscriminator(
    86→            stft_params=mstftcfg.stft_params,
    87→            in_channels=mstftcfg.in_channels,
    88→            out_channels=mstftcfg.out_channels,
    89→            kernel_sizes=mstftcfg.kernel_sizes,
    90→            channels=mstftcfg.channels,
    91→            max_downsample_channels=mstftcfg.max_downsample_channels,
    92→            downsample_scales=mstftcfg.downsample_scales,
    93→            use_weight_norm=mstftcfg.use_weight_norm,
    94→        )
    95→
    96→
    97→
    98→        # 单独编译需要优化的子模块
    99→        # self.CodecEnc = torch.compile(self.CodecEnc)
   100→        # self.generator.backbone = torch.compile(self.generator )
   101→        # self.mel_conv = torch.compile(self.mel_conv)
   102→
   103→        self.model_spk = model_spk .eval()
   104→
   105→        # self.semantic_model = AutoModel.from_pretrained("microsoft/wavlm-large")
   106→        # self.semantic_model.eval()
   107→        # self.semantic_model.requires_grad_(False)
   108→
   109→
   110→        self.fc_prior = nn.Linear(1024 + 1024, deccfg.vq_dim,   )
   111→        self.fc_post_a = nn.Linear(deccfg.vq_dim,  deccfg.hidden_dim )
   112→        self.fc_post_s = nn.Linear(deccfg.vq_dim,   1024)
   113→
   114→        self.SemanticDecoder_module = SemanticDecoder(1024, 1024, 1024)
   115→        self.SemanticEncoder_module = SemanticEncoder(1024, 1024, 1024)
   116→        self.semantic_model = Wav2Vec2BertModel.from_pretrained("facebook/w2v-bert-2.0", output_hidden_states=True)
   117→        self.semantic_model.eval()
   118→        self.semantic_model.requires_grad_(False)
   119→        # self.register_buffer('mel_basis', mel_basis)
   120→
   121→        # self.perception_model = AutoModel.from_pretrained("facebook/wav2vec2-large-xlsr-53")
   122→        # self.perception_model.eval()
   123→        # self.perception_model.requires_grad_(False)
   124→
   125→    def construct_criteria(self):
   126→        cfg = self.cfg.train
   127→        self.criteria = nn.ModuleDict()
   128→        if cfg.use_mel_loss:
   129→            self.criteria['mel_loss'] = MultiResolutionMelSpectrogramLoss(sample_rate=self.cfg.preprocess.audio.sr)
   130→        if cfg.use_stft_loss:
   131→            self.criteria['stft_loss'] = MultiResolutionSTFTLoss(
   132→                fft_sizes=cfg.stft_loss_params.fft_sizes,
   133→                hop_sizes=cfg.stft_loss_params.hop_sizes,
   134→                win_sizes=cfg.stft_loss_params.win_lengths
   135→            )
   136→        if cfg.use_feat_match_loss:
   137→            self.criteria['fm_loss'] = nn.L1Loss()
   138→        self.criteria['gan_loss'] = GANLoss()
   139→        self.criteria['l1_loss'] = nn.L1Loss()
   140→        self.criteria['l2_loss'] = nn.MSELoss()
   141→        print(self.criteria)
   142→
   143→
   144→
   145→
   146→    def forward(self, batch):
   147→        wav = batch['wav']
   148→        feats= batch['feats']
   149→
   150→        vq_emb = self.CodecEnc(wav.unsqueeze(1))

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

import os

import random
import hydra
import numpy as np
import librosa
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import pytorch_lightning as pl
from vq import CodecEncoder,  CodecDecoderVocos
from module import HiFiGANMultiPeriodDiscriminator, SpecDiscriminator
from criterions import GANLoss, MultiResolutionMelSpectrogramLoss, MultiResolutionSTFTLoss
from common.schedulers import WarmupLR
from transformers import AutoModel
from vq.module import SemanticDecoder,SemanticEncoder
from transformers import AutoFeatureExtractor, Wav2Vec2BertModel
import sys
sys.path.append('./eval_tools/tools/speaker_verification')    # We use wavlm_large_finetune as a vadidation metric during training, https://github.com/microsoft/UniSpeech/tree/main/downstreams/speaker_verification
from  verification import init_model
model_spk = init_model('wavlm_large','/aifs4su/data/zheny/models_fd_ckpt/wavlm_large_finetune.pth')

class CodecLightningModule(pl.LightningModule):
    def __init__(self, cfg):
        super().__init__()
        self.cfg = cfg
        self.ocwd = hydra.utils.get_original_cwd()
        self.construct_model()
        self.construct_criteria()
        self.save_hyperparameters()
        self.automatic_optimization = False

    def construct_model(self):
        # 初始化 Codec Encoder

        enccfg = self.cfg.model.codec_encoder

        self.CodecEnc = CodecEncoder(

            ngf=enccfg.ngf,
            up_ratios=enccfg.up_ratios,
            dilations=enccfg.dilations,
            hidden_dim=enccfg['hidden_dim'],
            depth=enccfg['depth'],
            heads=enccfg['heads'],
            pos_meb_dim=enccfg['pos_meb_dim'],
        )

        # 初始化 Codec Decoder
        deccfg = self.cfg.model.codec_decoder

        self.generator = CodecDecoderVocos(
            hidden_dim=deccfg.hidden_dim,
            depth=deccfg.depth,
            heads=deccfg.heads,
            pos_meb_dim=deccfg.pos_meb_dim,
            hop_length=320,
            vq_num_quantizers=deccfg.vq_num_quantizers,  # VQ 量化器数量
            vq_dim=deccfg.vq_dim,                   # VQ 维度
            vq_commit_weight=deccfg.vq_commit_weight,    # VQ 提交权重
            vq_weight_init=deccfg.vq_weight_init,         # VQ 权重初始化
            vq_full_commit_loss=deccfg.vq_full_commit_loss,  # 是否使用完整的提交损失
            codebook_size=deccfg.codebook_size,            # 码本大小
            codebook_dim=deccfg.codebook_dim ,              # 码本维度
                  # 隐藏层维度
        )

        # 初始化 MultiPeriod Discriminator
        mpdcfg = self.cfg.model.mpd
        self.discriminator = HiFiGANMultiPeriodDiscriminator(
            periods=mpdcfg.periods,
            max_downsample_channels=mpdcfg.max_downsample_channels,
            channels=mpdcfg.channels,
            channel_increasing_factor=mpdcfg.channel_increasing_factor,
        )

        # 初始化 Spectral Discriminator
        mstftcfg = self.cfg.model.mstft
        self.spec_discriminator = SpecDiscriminator(
            stft_params=mstftcfg.stft_params,
            in_channels=mstftcfg.in_channels,
            out_channels=mstftcfg.out_channels,
            kernel_sizes=mstftcfg.kernel_sizes,
            channels=mstftcfg.channels,
            max_downsample_channels=mstftcfg.max_downsample_channels,
            downsample_scales=mstftcfg.downsample_scales,
            use_weight_norm=mstftcfg.use_weight_norm,
        )

        # 单独编译需要优化的子模块
        # self.CodecEnc = torch.compile(self.CodecEnc)
        # self.generator.backbone = torch.compile(self.generator )
        # self.mel_conv = torch.compile(self.mel_conv)

        self.model_spk = model_spk .eval()

        # self.semantic_model = AutoModel.from_pretrained("microsoft/wavlm-large")
        # self.semantic_model.eval()
        # self.semantic_model.requires_grad_(False)

        self.fc_prior = nn.Linear(1024 + 1024, deccfg.vq_dim,   )
        self.fc_post_a = nn.Linear(deccfg.vq_dim,  deccfg.hidden_dim )
        self.fc_post_s = nn.Linear(deccfg.vq_dim,   1024)

        self.SemanticDecoder_module = SemanticDecoder(1024, 1024, 1024)
        self.SemanticEncoder_module = SemanticEncoder(1024, 1024, 1024)
        self.semantic_model = Wav2Vec2BertModel.from_pretrained("facebook/w2v-bert-2.0", output_hidden_states=True)
        self.semantic_model.eval()
        self.semantic_model.requires_grad_(False)
        # self.register_buffer('mel_basis', mel_basis)

        # self.perception_model = AutoModel.from_pretrained("facebook/wav2vec2-large-xlsr-53")
        # self.perception_model.eval()
        # self.perception_model.requires_grad_(False)

    def construct_criteria(self):
        cfg = self.cfg.train
        self.criteria = nn.ModuleDict()
        if cfg.use_mel_loss:
            self.criteria['mel_loss'] = MultiResolutionMelSpectrogramLoss(sample_rate=self.cfg.preprocess.audio.sr)
        if cfg.use_stft_loss:
            self.criteria['stft_loss'] = MultiResolutionSTFTLoss(
                fft_sizes=cfg.stft_loss_params.fft_sizes,
                hop_sizes=cfg.stft_loss_params.hop_sizes,
                win_sizes=cfg.stft_loss_params.win_lengths
            )
        if cfg.use_feat_match_loss:
            self.criteria['fm_loss'] = nn.L1Loss()
        self.criteria['gan_loss'] = GANLoss()
        self.criteria['l1_loss'] = nn.L1Loss()
        self.criteria['l2_loss'] = nn.MSELoss()
        print(self.criteria)

    def forward(self, batch):
        wav = batch['wav']
        feats= batch['feats']

        vq_emb = self.CodecEnc(wav.unsqueeze(1))


Let me look at the benchmark results file:


     1→{
     2→  "best_config": {
     3→    "batch_size": 256,
     4→    "decode_workers": 12,
     5→    "encode_workers": 12
     6→  },
     7→  "best_throughput": 29.13223594439401,
     8→  "all_results": [
     9→    {
    10→      "config": "b32_d8_e8",
    11→      "batch_size": 32,
    12→      "decode_workers": 8,
    13→      "encode_workers": 8,
    14→      "segments_per_sec": 26.04822452760495,
    15→      "outputs_per_sec": 45.85609378480118,
    16→      "gpu_memory_mb": 35.17138671875,
    17→      "elapsed_sec": 57.048034057952464
    18→    },
    19→    {
    20→      "config": "b64_d8_e8",
    21→      "batch_size": 64,
    22→      "decode_workers": 8,
    23→      "encode_workers": 8,
    24→      "segments_per_sec": 28.19994003449089,
    25→      "outputs_per_sec": 49.64403979153982,
    26→      "gpu_memory_mb": 35.17138671875,
    27→      "elapsed_sec": 52.6951475138776
    28→    },
    29→    {
    30→      "config": "b128_d8_e8",
    31→      "batch_size": 128,
    32→      "decode_workers": 8,
    33→      "encode_workers": 8,
    34→      "segments_per_sec": 28.902575750131035,
    35→      "outputs_per_sec": 50.880981266717896,
    36→      "gpu_memory_mb": 35.17138671875,
    37→      "elapsed_sec": 51.41410277225077
    38→    },
    39→    {
    40→      "config": "b64_d12_e8",
    41→      "batch_size": 64,
    42→      "decode_workers": 12,
    43→      "encode_workers": 8,
    44→      "segments_per_sec": 28.15212127967819,
    45→      "outputs_per_sec": 49.55985818818179,
    46→      "gpu_memory_mb": 35.17138671875,
    47→      "elapsed_sec": 52.78465467086062
    48→    },
    49→    {
    50→      "config": "b64_d8_e12",
    51→      "batch_size": 64,
    52→      "decode_workers": 8,
    53→      "encode_workers": 12,
    54→      "segments_per_sec": 27.99928466035629,
    55→      "outputs_per_sec": 49.2907999135209,
    56→      "gpu_memory_mb": 35.17138671875,
    57→      "elapsed_sec": 53.07278446666896
    58→    },
    59→    {
    60→      "config": "b64_d12_e12",
    61→      "batch_size": 64,
    62→      "decode_workers": 12,
    63→      "encode_workers": 12,
    64→      "segments_per_sec": 28.358666068997337,
    65→      "outputs_per_sec": 49.92346597341658,
    66→      "gpu_memory_mb": 35.17138671875,
    67→      "elapsed_sec": 52.40020797820762
    68→    },
    69→    {
    70→      "config": "b128_d12_e12",
    71→      "batch_size": 128,
    72→      "decode_workers": 12,
    73→      "encode_workers": 12,
    74→      "segments_per_sec": 28.62216984478426,
    75→      "outputs_per_sec": 50.387346106295844,
    76→      "gpu_memory_mb": 35.17138671875,
    77→      "elapsed_sec": 51.917796870693564
    78→    },
    79→    {
    80→      "config": "b64_d16_e16",
    81→      "batch_size": 64,
    82→      "decode_workers": 16,
    83→      "encode_workers": 16,
    84→      "segments_per_sec": 28.247406266468218,
    85→      "outputs_per_sec": 49.727600802880794,
    86→      "gpu_memory_mb": 35.17138671875,
    87→      "elapsed_sec": 52.606599911581725
    88→    },
    89→    {
    90→      "config": "b128_d16_e16",
    91→      "batch_size": 128,
    92→      "decode_workers": 16,
    93→      "encode_workers": 16,
    94→      "segments_per_sec": 28.414882507874836,
    95→      "outputs_per_sec": 50.02243111749702,
    96→      "gpu_memory_mb": 35.17138671875,
    97→      "elapsed_sec": 52.29653860395774
    98→    },
    99→    {
   100→      "config": "b256_d12_e12",
   101→      "batch_size": 256,
   102→      "decode_workers": 12,
   103→      "encode_workers": 12,
   104→      "segments_per_sec": 29.13223594439401,
   105→      "outputs_per_sec": 51.28528212014451,
   106→      "gpu_memory_mb": 35.17138671875,
   107→      "elapsed_sec": 51.00878637796268
   108→    }
   109→  ]
   110→}

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>


Now let me examine the remaining part of prepare_shards to understand the full pipeline:


   500→            search_low = offset + cfg.min_sec
   501→            search_high = min(offset + cfg.max_sec, latest_cut)
   502→        boundary = find_silence_boundary(
   503→            times,
   504→            rms,
   505→            search_low,
   506→            search_high,
   507→            threshold_rms,
   508→            median_db,
   509→            silence_cfg.db_drop,
   510→        )
   511→        if boundary is None:
   512→            if not windows:
   513→                return [(0.0, duration_sec)]
   514→            windows.append((offset, duration_sec - offset))
   515→            return windows
   516→        windows.append((offset, boundary - offset))
   517→        offset = boundary
   518→
   519→    windows.append((offset, duration_sec - offset))
   520→    return windows
   521→
   522→
   523→def make_segment_seed(base_seed: int, video_id: str, speaker: str, index: int) -> int:
   524→    key = f"{video_id}-{speaker}-{index}-{base_seed}"
   525→    return zlib.adler32(key.encode("utf-8")) & 0xFFFFFFFF
   526→
   527→
   528→def parse_segment_filename(filename: str) -> Optional[tuple[str, float]]:
   529→    """Parse segment filename to extract speaker and start time.
   530→
   531→    Filename format: segments/SPEAKER_XX_NNNN_start-end.flac
   532→    Example: segments/SPEAKER_00_0007_35.10-37.49.flac
   533→
   534→    Returns (speaker, start_time) for matching against metadata.
   535→    """
   536→    base = filename.split("/")[-1]
   537→    if not base.endswith(".flac"):
   538→        return None
   539→
   540→    # Remove .flac extension
   541→    base = base[:-5]
   542→
   543→    # Try to extract time range (last part after final underscore)
   544→    parts = base.split("_")
   545→    if len(parts) < 3:
   546→        return None
   547→
   548→    # Speaker is everything except last two parts (index and time range)
   549→    speaker = "_".join(parts[:-2])
   550→
   551→    # Time range is the last part: "35.10-37.49"
   552→    time_range = parts[-1]
   553→    if "-" not in time_range:
   554→        return None
   555→
   556→    try:
   557→        start_str = time_range.split("-")[0]
   558→        start_time = float(start_str)
   559→    except (ValueError, IndexError):
   560→        return None
   561→
   562→    return speaker, start_time
   563→
   564→
   565→def process_segment(
   566→    video_id: str,
   567→    language: str,
   568→    segment_bytes: bytes,
   569→    segment_meta: dict[str, Any],
   570→    speaker: str,
   571→    index: int,
   572→    chunk_cfg: ChunkConfig,
   573→    silence_cfg: SilenceConfig,
   574→    rng_seed: int,
   575→    output_format: str,
   576→) -> tuple[list[SegmentOutput], dict[str, int]]:
   577→    with timer("segment_total"):
   578→        stats = {
   579→            "dropped_too_short": 0,
   580→            "dropped_too_long": 0,
   581→            "dropped_clipped": 0,
   582→            "chunks_emitted": 0,
   583→            "segments_chunked": 0,
   584→        }
   585→        outputs: list[SegmentOutput] = []
   586→
   587→        with timer("flac_decode"):
   588→            try:
   589→                audio, sr = sf.read(io.BytesIO(segment_bytes), dtype="float32")
   590→            except Exception:
   591→                return outputs, stats
   592→
   593→        if audio.ndim > 1:
   594→            audio = np.mean(audio, axis=1)
   595→
   596→        rng = random.Random(rng_seed)
   597→        windows = make_chunk_windows(audio, sr, chunk_cfg, silence_cfg, rng)
   598→
   599→        music_stats = segment_meta.get("music_stats") or {}
   600→        has_music = bool(music_stats.get("has_music", False))
   601→
   602→        def emit_outputs(
   603→            audio_slice: np.ndarray,
   604→            metrics: dict[str, float],
   605→            sample_id: str,
   606→            segment_index: int,
   607→            chunk_index: Optional[int],
   608→            start: Optional[float],
   609→            end: Optional[float],
   610→            original_start: Optional[float],
   611→            original_end: Optional[float],
   612→            target_srs: list[int],
   613→        ) -> None:
   614→            slice_duration = float(audio_slice.shape[0] / sr)
   615→            bucket = duration_bucket(slice_duration)
   616→            if bucket is None:
   617→                if slice_duration < 3.0:
   618→                    stats["dropped_too_short"] += 1
   619→                else:
   620→                    stats["dropped_too_long"] += 1
   621→                return
   622→            if metrics["clipping_fraction"] > 0.1:
   623→                stats["dropped_clipped"] += 1
   624→                return
   625→
   626→            duration_bucket_name, base_weight = bucket
   627→            sampling_weight = base_weight * (0.5 if has_music else 1.0)
   628→            quality_score = compute_quality_score(metrics, has_music)
   629→            effective_nyquist = metrics["effective_nyquist_hz"]
   630→
   631→            base_metadata = {
   632→                "id": sample_id,
   633→                "video_id": video_id,
   634→                "language": language,
   635→                "segment_index": segment_index,
   636→                "chunk_index": chunk_index,
   637→                "start": start,
   638→                "end": end,
   639→                "original_start": original_start,
   640→                "original_end": original_end,
   641→                "duration_sec": slice_duration,
   642→                "native_sr": sr,
   643→                "effective_nyquist_hz": effective_nyquist,
   644→                "spectral_rolloff_hz": metrics["spectral_rolloff_hz"],
   645→                "high_freq_energy_ratio": metrics["high_freq_energy_ratio"],
   646→                "is_truly_wideband": bool(effective_nyquist >= 12000),
   647→                "native_sr_bucket": classify_native_bucket(effective_nyquist),
   648→                "loudness_lufs": metrics["loudness_lufs"],
   649→                "loudness_rms_db": metrics["loudness_rms_db"],
   650→                "clipping_fraction": metrics["clipping_fraction"],
   651→                "snr_proxy": metrics["snr_proxy"],
   652→                "quality_score": quality_score,
   653→                "music_prob": float(music_stats.get("music_mean", 0.0)),
   654→                "music_stats": music_stats,
   655→                "speaker": speaker,
   656→                # VAD regions: Full duration placeholder; random 6s crops happen during training.
   657→                "vad_speech_regions": [[0.0, slice_duration]],
   658→                "duration_bucket": duration_bucket_name,
   659→                "sampling_weight": sampling_weight,
   660→            }
   661→
   662→            for target_sr in target_srs:
   663→                resampled = resample_audio(audio_slice, sr, target_sr)
   664→                resampled = np.clip(resampled, -1.0, 1.0)
   665→                audio_bytes = encode_audio(resampled, target_sr, output_format)
   666→                meta = dict(base_metadata)
   667→                meta["sample_rate"] = target_sr
   668→                meta["resample_target_sr"] = target_sr
   669→                outputs.append(SegmentOutput(target_sr, audio_bytes, meta))
   670→
   671→            if target_srs == [16000]:
   672→                stats["chunks_emitted"] += 1
   673→
   674→        sample_id_base = f"{video_id}__{speaker}_{index:04d}"
   675→        use_chunk_suffix = len(windows) > 1
   676→        if use_chunk_suffix:
   677→            stats["segments_chunked"] += 1
   678→
   679→        segment_start = segment_meta.get("start")
   680→        segment_end = segment_meta.get("end")
   681→        original_start = segment_meta.get("original_start")
   682→        original_end = segment_meta.get("original_end")
   683→
   684→        # Always emit 16k outputs; for long segments, emit chunked windows to boost sample count.
   685→        for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
   686→            start_idx = int(round(offset_sec * sr))
   687→            end_idx = int(round((offset_sec + chunk_len_sec) * sr))
   688→            audio_slice = audio[start_idx:end_idx]
   689→            if audio_slice.size == 0:
   690→                continue
   691→            slice_duration = float(audio_slice.shape[0] / sr)
   692→            slice_metrics = compute_audio_metrics(audio_slice, sr)
   693→
   694→            chunk_start = float(segment_start) + offset_sec if segment_start is not None else None
   695→            chunk_end = chunk_start + slice_duration if chunk_start is not None else None
   696→            chunk_original_start = (
   697→                float(original_start) + offset_sec if original_start is not None else None
   698→            )
   699→            chunk_original_end = (
   700→                chunk_original_start + slice_duration
   701→                if chunk_original_start is not None
   702→                else None
   703→            )
   704→
   705→            sample_id = sample_id_base
   706→            chunk_index = None
   707→            if use_chunk_suffix:
   708→                sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
   709→                chunk_index = chunk_idx
   710→
   711→            emit_outputs(
   712→                audio_slice,
   713→                slice_metrics,
   714→                sample_id,
   715→                index,
   716→                chunk_index,
   717→                chunk_start,
   718→                chunk_end,
   719→                chunk_original_start,
   720→                chunk_original_end,
   721→                [16000],
   722→            )
   723→
   724→        # Optional high-rate buckets: use full segment unless configured otherwise.
   725→        full_metrics = compute_audio_metrics(audio, sr)
   726→        effective_nyquist = full_metrics["effective_nyquist_hz"]
   727→        targets_high: list[int] = []
   728→        if effective_nyquist >= 8000:
   729→            targets_high.append(24000)
   730→        if effective_nyquist >= 12000:
   731→            targets_high.append(32000)
   732→        if effective_nyquist >= 16000:
   733→            targets_high.append(44100)
   734→
   735→        if targets_high:
   736→            if chunk_cfg.all_buckets and use_chunk_suffix:
   737→                for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
   738→                    start_idx = int(round(offset_sec * sr))
   739→                    end_idx = int(round((offset_sec + chunk_len_sec) * sr))
   740→                    audio_slice = audio[start_idx:end_idx]
   741→                    if audio_slice.size == 0:
   742→                        continue
   743→                    slice_duration = float(audio_slice.shape[0] / sr)
   744→                    slice_metrics = compute_audio_metrics(audio_slice, sr)
   745→                    slice_effective_nyquist = slice_metrics["effective_nyquist_hz"]
   746→                    slice_targets: list[int] = []
   747→                    if slice_effective_nyquist >= 8000:
   748→                        slice_targets.append(24000)
   749→                    if slice_effective_nyquist >= 12000:

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

            search_low = offset + cfg.min_sec
            search_high = min(offset + cfg.max_sec, latest_cut)
        boundary = find_silence_boundary(
            times,
            rms,
            search_low,
            search_high,
            threshold_rms,
            median_db,
            silence_cfg.db_drop,
        )
        if boundary is None:
            if not windows:
                return [(0.0, duration_sec)]
            windows.append((offset, duration_sec - offset))
            return windows
        windows.append((offset, boundary - offset))
        offset = boundary

    windows.append((offset, duration_sec - offset))
    return windows

def make_segment_seed(base_seed: int, video_id: str, speaker: str, index: int) -> int:
    key = f"{video_id}-{speaker}-{index}-{base_seed}"
    return zlib.adler32(key.encode("utf-8")) & 0xFFFFFFFF

def parse_segment_filename(filename: str) -> Optional[tuple[str, float]]:
    """Parse segment filename to extract speaker and start time.

    Filename format: segments/SPEAKER_XX_NNNN_start-end.flac
    Example: segments/SPEAKER_00_0007_35.10-37.49.flac

    Returns (speaker, start_time) for matching against metadata.
    """
    base = filename.split("/")[-1]
    if not base.endswith(".flac"):
        return None

    # Remove .flac extension
    base = base[:-5]

    # Try to extract time range (last part after final underscore)
    parts = base.split("_")
    if len(parts) < 3:
        return None

    # Speaker is everything except last two parts (index and time range)
    speaker = "_".join(parts[:-2])

    # Time range is the last part: "35.10-37.49"
    time_range = parts[-1]
    if "-" not in time_range:
        return None

    try:
        start_str = time_range.split("-")[0]
        start_time = float(start_str)
    except (ValueError, IndexError):
        return None

    return speaker, start_time

def process_segment(
    video_id: str,
    language: str,
    segment_bytes: bytes,
    segment_meta: dict[str, Any],
    speaker: str,
    index: int,
    chunk_cfg: ChunkConfig,
    silence_cfg: SilenceConfig,
    rng_seed: int,
    output_format: str,
) -> tuple[list[SegmentOutput], dict[str, int]]:
    with timer("segment_total"):
        stats = {
            "dropped_too_short": 0,
            "dropped_too_long": 0,
            "dropped_clipped": 0,
            "chunks_emitted": 0,
            "segments_chunked": 0,
        }
        outputs: list[SegmentOutput] = []

        with timer("flac_decode"):
            try:
                audio, sr = sf.read(io.BytesIO(segment_bytes), dtype="float32")
            except Exception:
                return outputs, stats

        if audio.ndim > 1:
            audio = np.mean(audio, axis=1)

        rng = random.Random(rng_seed)
        windows = make_chunk_windows(audio, sr, chunk_cfg, silence_cfg, rng)

        music_stats = segment_meta.get("music_stats") or {}
        has_music = bool(music_stats.get("has_music", False))

        def emit_outputs(
            audio_slice: np.ndarray,
            metrics: dict[str, float],
            sample_id: str,
            segment_index: int,
            chunk_index: Optional[int],
            start: Optional[float],
            end: Optional[float],
            original_start: Optional[float],
            original_end: Optional[float],
            target_srs: list[int],
        ) -> None:
            slice_duration = float(audio_slice.shape[0] / sr)
            bucket = duration_bucket(slice_duration)
            if bucket is None:
                if slice_duration < 3.0:
                    stats["dropped_too_short"] += 1
                else:
                    stats["dropped_too_long"] += 1
                return
            if metrics["clipping_fraction"] > 0.1:
                stats["dropped_clipped"] += 1
                return

            duration_bucket_name, base_weight = bucket
            sampling_weight = base_weight * (0.5 if has_music else 1.0)
            quality_score = compute_quality_score(metrics, has_music)
            effective_nyquist = metrics["effective_nyquist_hz"]

            base_metadata = {
                "id": sample_id,
                "video_id": video_id,
                "language": language,
                "segment_index": segment_index,
                "chunk_index": chunk_index,
                "start": start,
                "end": end,
                "original_start": original_start,
                "original_end": original_end,
                "duration_sec": slice_duration,
                "native_sr": sr,
                "effective_nyquist_hz": effective_nyquist,
                "spectral_rolloff_hz": metrics["spectral_rolloff_hz"],
                "high_freq_energy_ratio": metrics["high_freq_energy_ratio"],
                "is_truly_wideband": bool(effective_nyquist >= 12000),
                "native_sr_bucket": classify_native_bucket(effective_nyquist),
                "loudness_lufs": metrics["loudness_lufs"],
                "loudness_rms_db": metrics["loudness_rms_db"],
                "clipping_fraction": metrics["clipping_fraction"],
                "snr_proxy": metrics["snr_proxy"],
                "quality_score": quality_score,
                "music_prob": float(music_stats.get("music_mean", 0.0)),
                "music_stats": music_stats,
                "speaker": speaker,
                # VAD regions: Full duration placeholder; random 6s crops happen during training.
                "vad_speech_regions": [[0.0, slice_duration]],
                "duration_bucket": duration_bucket_name,
                "sampling_weight": sampling_weight,
            }

            for target_sr in target_srs:
                resampled = resample_audio(audio_slice, sr, target_sr)
                resampled = np.clip(resampled, -1.0, 1.0)
                audio_bytes = encode_audio(resampled, target_sr, output_format)
                meta = dict(base_metadata)
                meta["sample_rate"] = target_sr
                meta["resample_target_sr"] = target_sr
                outputs.append(SegmentOutput(target_sr, audio_bytes, meta))

            if target_srs == [16000]:
                stats["chunks_emitted"] += 1

        sample_id_base = f"{video_id}__{speaker}_{index:04d}"
        use_chunk_suffix = len(windows) > 1
        if use_chunk_suffix:
            stats["segments_chunked"] += 1

        segment_start = segment_meta.get("start")
        segment_end = segment_meta.get("end")
        original_start = segment_meta.get("original_start")
        original_end = segment_meta.get("original_end")

        # Always emit 16k outputs; for long segments, emit chunked windows to boost sample count.
        for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
            start_idx = int(round(offset_sec * sr))
            end_idx = int(round((offset_sec + chunk_len_sec) * sr))
            audio_slice = audio[start_idx:end_idx]
            if audio_slice.size == 0:
                continue
            slice_duration = float(audio_slice.shape[0] / sr)
            slice_metrics = compute_audio_metrics(audio_slice, sr)

            chunk_start = float(segment_start) + offset_sec if segment_start is not None else None
            chunk_end = chunk_start + slice_duration if chunk_start is not None else None
            chunk_original_start = (
                float(original_start) + offset_sec if original_start is not None else None
            )
            chunk_original_end = (
                chunk_original_start + slice_duration
                if chunk_original_start is not None
                else None
            )

            sample_id = sample_id_base
            chunk_index = None
            if use_chunk_suffix:
                sample_id = f"{sample_id_base}_c{chunk_idx:02d}"
                chunk_index = chunk_idx

            emit_outputs(
                audio_slice,
                slice_metrics,
                sample_id,
                index,
                chunk_index,
                chunk_start,
                chunk_end,
                chunk_original_start,
                chunk_original_end,
                [16000],
            )

        # Optional high-rate buckets: use full segment unless configured otherwise.
        full_metrics = compute_audio_metrics(audio, sr)
        effective_nyquist = full_metrics["effective_nyquist_hz"]
        targets_high: list[int] = []
        if effective_nyquist >= 8000:
            targets_high.append(24000)
        if effective_nyquist >= 12000:
            targets_high.append(32000)
        if effective_nyquist >= 16000:
            targets_high.append(44100)

        if targets_high:
            if chunk_cfg.all_buckets and use_chunk_suffix:
                for chunk_idx, (offset_sec, chunk_len_sec) in enumerate(windows):
                    start_idx = int(round(offset_sec * sr))
                    end_idx = int(round((offset_sec + chunk_len_sec) * sr))
                    audio_slice = audio[start_idx:end_idx]
                    if audio_slice.size == 0:
                        continue
                    slice_duration = float(audio_slice.shape[0] / sr)
                    slice_metrics = compute_audio_metrics(audio_slice, sr)
                    slice_effective_nyquist = slice_metrics["effective_nyquist_hz"]
                    slice_targets: list[int] = []
                    if slice_effective_nyquist >= 8000:
                        slice_targets.append(24000)
                    if slice_effective_nyquist >= 12000:


Perfect! Now let me create a comprehensive summary document. Let me verify the directory structure once more:


find /home/ubuntu/maya3_prepare -type f -name "*.py" | grep -E "(prepare_shards|fast_audio|benchmark)" | sort


 1589 /home/ubuntu/maya3_prepare/scripts/prepare_shards.py
  613 /home/ubuntu/maya3_prepare/scripts/fast_audio_pipeline.py
  447 /home/ubuntu/maya3_prepare/scripts/fast_audio_pipeline_v2.py
 2649 total


Now let me create a comprehensive report by examining a few more critical sections: