     1→#!/usr/bin/env python3
     2→"""Phase 6: Preflight checks before smoke training.
     3→
     4→1. 10k random URI decode check — zero failures
     5→2. Zero runtime resample events
     6→3. Verify dataset_info.json and parquet are valid
     7→"""
     8→
     9→import json
    10→import os
    11→import sys
    12→import time
    13→from concurrent.futures import ProcessPoolExecutor, as_completed
    14→from pathlib import Path
    15→
    16→import numpy as np
    17→import pyarrow.parquet as pq
    18→
    19→os.environ["OMP_NUM_THREADS"] = "1"
    20→os.environ["MKL_NUM_THREADS"] = "1"
    21→os.environ["TORCH_NUM_THREADS"] = "1"
    22→
    23→DATASET_DIR = "/root/gemini-asr/lf_asr/artifacts/phase4"
    24→N_DECODE_CHECK = 10000
    25→WORKERS = 32
    26→SEED = 42
    27→
    28→
    29→def decode_uri(args):
    30→    """Decode a single URI and verify SR=16000."""
    31→    uri, expected_sr = args
    32→    try:
    33→        sys.path.insert(0, "/root/gemini-asr/lf_asr/vendor/LlamaFactory/src")
    34→        from llamafactory.data.mm_plugin import _load_audio_from_tar_uri
    35→        audio, sr = _load_audio_from_tar_uri(uri)
    36→        if sr != expected_sr:
    37→            return {"success": False, "error": f"SR mismatch: {sr} != {expected_sr}", "uri": uri}
    38→        if audio.shape[-1] == 0:
    39→            return {"success": False, "error": "Empty audio", "uri": uri}
    40→        return {"success": True, "sr": sr, "samples": audio.shape[-1]}
    41→    except Exception as e:
    42→        return {"success": False, "error": str(e), "uri": uri}
    43→
    44→
    45→def decode_uri_with_fallback(uri: str, expected_sr: int = 16000) -> dict:
    46→    """Verify plugin fallback path can recover from decode errors."""
    47→    try:
    48→        sys.path.insert(0, "/root/gemini-asr/lf_asr/vendor/LlamaFactory/src")
    49→        from llamafactory.data.mm_plugin import get_mm_plugin
    50→
    51→        os.environ["LLAMAFACTORY_AUDIO_DECODE_FALLBACK"] = "1"
    52→        os.environ["LLAMAFACTORY_AUDIO_DECODE_FALLBACK_SEC"] = "1.0"
    53→        plugin = get_mm_plugin("gemma3n")
    54→        out = plugin._regularize_audios([uri], sampling_rate=expected_sr)
    55→        audio = out["audios"][0]
    56→        if audio.shape[0] == 0:
    57→            return {"success": False, "error": "fallback returned empty audio", "uri": uri}
    58→        return {"success": True, "samples": int(audio.shape[0]), "uri": uri}
    59→    except Exception as e:
    60→        return {"success": False, "error": str(e), "uri": uri}
    61→
    62→
    63→def main():
    64→    print("Phase 6 Preflight Checks")
    65→    print("=" * 60)
    66→
    67→    # Check 1: dataset_info.json exists
    68→    info_path = os.path.join(DATASET_DIR, "dataset_info.json")
    69→    assert os.path.exists(info_path), f"Missing {info_path}"
    70→    with open(info_path) as f:
    71→        info = json.load(f)
    72→    assert "asr_train" in info
    73→    print("[PASS] dataset_info.json valid")
    74→
    75→    # Check 2: parquet files exist and have expected columns.
    76→    # phase4 may intentionally omit test.parquet (all-data training policy).
    77→    required_splits = ["train", "dev"]
    78→    optional_splits = ["test"]
    79→    split_paths: dict[str, Path] = {}
    80→    expected_cols = {"instruction", "input", "output", "audios"}
    81→
    82→    for split in required_splits + optional_splits:
    83→        path = Path(DATASET_DIR) / f"{split}.parquet"
    84→        if split in required_splits:
    85→            assert path.exists(), f"Missing required {path}"
    86→        elif not path.exists():
    87→            print(f"[INFO] Optional {path.name} not found; continuing.")
    88→            continue
    89→
    90→        pf = pq.ParquetFile(path)
    91→        cols = set(pf.schema_arrow.names)
    92→        missing = expected_cols - cols
    93→        assert not missing, f"{path} missing columns: {sorted(missing)}"
    94→        assert pf.metadata.num_rows > 0, f"{path} is empty"
    95→
    96→        # Verify first row format without loading full file.
    97→        first_rg = pf.read_row_group(0, columns=["instruction", "audios"])
    98→        instruction = first_rg["instruction"][0].as_py()
    99→        audios_val = first_rg["audios"][0].as_py()
   100→        assert isinstance(instruction, str) and instruction.startswith("<audio>")
   101→        assert hasattr(audios_val, "__iter__") and len(audios_val) > 0
   102→        assert str(audios_val[0]).startswith("tar://")
   103→        split_paths[split] = path
   104→        print(f"[PASS] {split}.parquet: {pf.metadata.num_rows:,} rows, columns OK")
   105→
   106→    # Check 3: 10k random URI decode
   107→    print(f"\nDecoding {N_DECODE_CHECK:,} random URIs with {WORKERS} workers...")
   108→    train_pf = pq.ParquetFile(split_paths["train"])
   109→    rng = np.random.default_rng(SEED)
   110→    num_rgs = train_pf.metadata.num_row_groups
   111→    rg_sizes = [train_pf.metadata.row_group(i).num_rows for i in range(num_rgs)]
   112→
   113→    sample_n = min(N_DECODE_CHECK, train_pf.metadata.num_rows)
   114→    picks_by_rg: dict[int, list[int]] = {}
   115→    for _ in range(sample_n):
   116→        rg = int(rng.integers(0, num_rgs))
   117→        row_idx = int(rng.integers(0, rg_sizes[rg]))
   118→        picks_by_rg.setdefault(rg, []).append(row_idx)
   119→
   120→    uris: list[str] = []
   121→    for rg, row_indices in picks_by_rg.items():
   122→        tbl = train_pf.read_row_group(rg, columns=["audios"])
   123→        col = tbl["audios"]
   124→        for row_idx in row_indices:
   125→            audios_val = col[row_idx].as_py()
   126→            uri = audios_val[0] if isinstance(audios_val, list) else audios_val
   127→            uris.append(uri)
   128→
   129→    assert len(uris) == sample_n, f"URI sampling mismatch: {len(uris)} != {sample_n}"
   130→    tasks = [(uri, 16000) for uri in uris]
   131→
   132→    t_start = time.perf_counter()
   133→    results = []
   134→    with ProcessPoolExecutor(max_workers=WORKERS) as executor:
   135→        futures = {executor.submit(decode_uri, t): t for t in tasks}
   136→        for f in as_completed(futures):
   137→            results.append(f.result())
   138→    t_end = time.perf_counter()
   139→
   140→    ok = [r for r in results if r["success"]]
   141→    fail = [r for r in results if not r["success"]]
   142→
   143→    print(f"  Decoded {len(ok):,} OK, {len(fail)} failed in {t_end - t_start:.1f}s")
   144→    if fail:
   145→        print(f"  FAILURES:")
   146→        for f_item in fail[:10]:
   147→            print(f"    {f_item['error']}: {f_item['uri'][:80]}...")
   148→
   149→    # Check for resample (all should be 16000)
   150→    sr_mismatch = [r for r in fail if "SR mismatch" in r.get("error", "")]
   151→    print(f"  SR mismatches (runtime resample events): {len(sr_mismatch)}")
   152→
   153→    fallback_checked = 0
   154→    fallback_failed = 0
   155→    if fail:
   156→        print("  Verifying fallback-path recovery on failed URIs...")
   157→        fallback_results = [decode_uri_with_fallback(r["uri"]) for r in fail]
   158→        fallback_checked = len(fallback_results)
   159→        fallback_failed = sum(1 for r in fallback_results if not r["success"])
   160→        print(f"  Fallback recovery: {fallback_checked - fallback_failed}/{fallback_checked} OK")
   161→
   162→    # Verdict
   163→    print(f"\n{'='*60}")
   164→    all_pass_strict = len(fail) == 0 and len(sr_mismatch) == 0
   165→    all_pass_with_fallback = len(sr_mismatch) == 0 and fallback_failed == 0
   166→    if all_pass_strict:
   167→        print("PREFLIGHT: ALL CHECKS PASSED")
   168→    elif all_pass_with_fallback:
   169→        print("PREFLIGHT: PASS_WITH_FALLBACK (strict decode had recoverable corrupt samples)")
   170→    else:
   171→        print(
   172→            "PREFLIGHT: FAILED "
   173→            f"({len(fail)} strict decode failures, {len(sr_mismatch)} SR mismatches, "
   174→            f"{fallback_failed} fallback failures)"
   175→        )
   176→
   177→    # Save report
   178→    report = {
   179→        "decode_checked": len(results),
   180→        "decode_ok": len(ok),
   181→        "decode_failed": len(fail),
   182→        "sr_mismatches": len(sr_mismatch),
   183→        "fallback_checked": fallback_checked,
   184→        "fallback_failed": fallback_failed,
   185→        "wall_time_s": round(t_end - t_start, 2),
   186→        "all_pass_strict": all_pass_strict,
   187→        "all_pass_with_fallback": all_pass_with_fallback,
   188→    }
   189→    with open(os.path.join(DATASET_DIR, "../phase6_preflight.json"), "w") as f:
   190→        json.dump(report, f, indent=2)
   191→
   192→
   193→if __name__ == "__main__":
   194→    main()
   195→


Perfect. Now let me create a comprehensive summary document by reading a few more key files:


     1→# Copyright 2025 the LlamaFactory team.
     2→#
     3→# Licensed under the Apache License, Version 2.0 (the "License");
     4→# you may not use this file except in compliance with the License.
     5→# You may obtain a copy of the License at
     6→#
     7→#     http://www.apache.org/licenses/LICENSE-2.0
     8→#
     9→# Unless required by applicable law or agreed to in writing, software
    10→# distributed under the License is distributed on an "AS IS" BASIS,
    11→# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12→# See the License for the specific language governing permissions and
    13→# limitations under the License.
    14→
    15→import re
    16→from copy import deepcopy
    17→from dataclasses import dataclass
    18→from typing import TYPE_CHECKING, Optional, Union
    19→
    20→from typing_extensions import override
    21→
    22→from ..extras import logging
    23→from .data_utils import Role
    24→from .formatter import EmptyFormatter, FunctionFormatter, StringFormatter, ToolFormatter
    25→from .mm_plugin import get_mm_plugin
    26→
    27→
    28→if TYPE_CHECKING:
    29→    from transformers import PreTrainedTokenizer
    30→
    31→    from ..hparams import DataArguments
    32→    from .formatter import SLOTS, Formatter
    33→    from .mm_plugin import BasePlugin
    34→    from .tool_utils import FunctionCall
    35→
    36→
    37→logger = logging.get_logger(__name__)
    38→
    39→
    40→@dataclass
    41→class Template:
    42→    format_user: "Formatter"
    43→    format_assistant: "Formatter"
    44→    format_system: "Formatter"
    45→    format_function: "Formatter"
    46→    format_observation: "Formatter"
    47→    format_tools: "Formatter"
    48→    format_prefix: "Formatter"
    49→    default_system: str
    50→    stop_words: list[str]
    51→    thought_words: tuple[str, str]
    52→    tool_call_words: tuple[str, str]
    53→    efficient_eos: bool
    54→    replace_eos: bool
    55→    replace_jinja_template: bool
    56→    enable_thinking: Optional[bool]
    57→    mm_plugin: "BasePlugin"
    58→
    59→    def encode_oneturn(
    60→        self,
    61→        tokenizer: "PreTrainedTokenizer",
    62→        messages: list[dict[str, str]],
    63→        system: Optional[str] = None,
    64→        tools: Optional[str] = None,
    65→    ) -> tuple[list[int], list[int]]:
    66→        r"""Return a single pair of token ids representing prompt and response respectively."""
    67→        encoded_messages = self._encode(tokenizer, messages, system, tools)
    68→        prompt_ids = []
    69→        for encoded_ids in encoded_messages[:-1]:
    70→            prompt_ids += encoded_ids
    71→
    72→        response_ids = encoded_messages[-1]
    73→        return prompt_ids, response_ids
    74→
    75→    def encode_multiturn(
    76→        self,
    77→        tokenizer: "PreTrainedTokenizer",
    78→        messages: list[dict[str, str]],
    79→        system: Optional[str] = None,
    80→        tools: Optional[str] = None,
    81→    ) -> list[tuple[list[int], list[int]]]:
    82→        r"""Return multiple pairs of token ids representing prompts and responses respectively."""
    83→        encoded_messages = self._encode(tokenizer, messages, system, tools)
    84→        return [(encoded_messages[i], encoded_messages[i + 1]) for i in range(0, len(encoded_messages), 2)]
    85→
    86→    def extract_tool(self, content: str) -> Union[str, list["FunctionCall"]]:
    87→        r"""Extract tool message."""
    88→        return self.format_tools.extract(content)
    89→
    90→    def get_stop_token_ids(self, tokenizer: "PreTrainedTokenizer") -> list[int]:
    91→        r"""Return stop token ids."""
    92→        stop_token_ids = {tokenizer.eos_token_id}
    93→        for token in self.stop_words:
    94→            stop_token_ids.add(tokenizer.convert_tokens_to_ids(token))
    95→
    96→        return list(stop_token_ids)
    97→
    98→    def add_thought(self, content: str = "") -> str:
    99→        r"""Add empty thought to assistant message."""
   100→        return f"{self.thought_words[0]}{self.thought_words[1]}" + content


Now let me check the git history for context:


Bash

cd /home/ubuntu/training/gemma3n-E2B-asr-ft && git log --oneline | head -20


d70c21e Gemma3n-E2B ASR full finetune pipeline


Perfect. Now I have comprehensive understanding. Let me create a detailed summary report:

## COMPREHENSIVE GEMMA3N-E2B-ASR-FT REPOSITORY ANALYSIS

### 1. OVERALL STRUCTURE

The repository at `/home/ubuntu/training/gemma3n-E2B-asr-ft/` is an ASR (Automatic Speech Recognition) fine-tuning pipeline built on top of LlamaFactory. Directory structure:

```
gemma3n-E2B-asr-ft/
├── artifacts/              # Pipeline outputs (phase2, phase4, phase6)
├── benchmarks/            # Benchmark manifests and results
│   └── phase1/           # Raw audio pipeline benchmarks
├── configs/              # Training configurations
│   ├── gemma3n_asr_full_ft.yaml      # Full fine-tune (8 batch size)
│   ├── gemma3n_asr_lora_smoke.yaml   # LoRA smoke test (4 batch size)
│   ├── smoke_100step.yaml
│   └── smoke_500step.yaml
├── docs/                 # Documentation and runbooks
│   ├── overnight_plan.md       # Phases 3-6 execution plan
│   ├── phase2_runbook.md       # Audio conversion pipeline
│   └── perf_contract.md        # Performance benchmarks
├── scripts/              # Launch and monitoring scripts
│   ├── launch_smoke.sh
│   ├── launch_full_ft.sh
│   ├── watch_training.sh
│   └── push_ckpt_r2.sh
├── tools/                # Phase-by-phase pipeline tools
├── vendor/LlamaFactory/  # Local copy of LlamaFactory (model training framework)
└── .git/
```

### 2. MODEL ARCHITECTURE

**Base Model:** Gemma-3N (Google Gemma 3 Nano multimodal model)
- **Architecture Type:** Vision-Language Model with Audio Support (multimodal)
- **Framework:** LlamaFactory (fine-tuning framework)
- **Model Location:** `/workspace/gemini-asr/gemma-model`
- **Task:** Audio Transcription (Speech-to-Text)

**Key Multimodal Components:**
- Vision tower: For image/video processing
- Audio encoder: For audio input processing
- Multimodal projector: Fuses audio features with text tokens
- Template: `gemma3n` with audio soft tokens (`<audio_soft_token>`)

**Audio Input Format:**
- Frequency: 16 kHz (pre-converted, no runtime resampling)
- Channels: Mono
- Format: FLAC (lossless compression)
- Input Token: `<audio>` in instruction text

### 3. INFERENCE SUPPORT & MODEL LOADING

**Inference Engine:** HuggingFace Transformers (with LlamaFactory wrapper)

**Key Classes:**
1. **ChatModel** (`/home/ubuntu/training/gemma3n-E2B-asr-ft/vendor/LlamaFactory/src/llamafactory/chat/chat_model.py`)
   - Main inference interface
   - Supports sync: `chat()`, `stream_chat()`, `get_scores()`
   - Supports async: `achat()`, `astream_chat()`, `aget_scores()`
   - Accepts audio via `audios` parameter

2. **HuggingfaceEngine** (hf_engine.py)
   - Loads tokenizer: `load_tokenizer(model_args)`
   - Loads model: `load_model(tokenizer, model_args, ...)`
   - Processes multimodal inputs (images, videos, audios)
   - Audio is passed as list of audio paths or tensor data

3. **Audio Decoding Plugin** (`mm_plugin.py`)
   - **tar:// URI Support:** `_load_audio_from_tar_uri(uri)` → (torch.Tensor, sample_rate)
   - **URI Format:** `tar:///path/to/audio.tar|offset|nbytes`
   - **Implementation:** Uses `os.pread()` for thread-safe random access with FD caching
   - **Fallback:** Zero-audio tensor (1 second at target SR) on decode errors

**Audio Loading Pipeline:**
```python
# From mm_plugin.py
def _load_audio_from_tar_uri(uri: str) -> tuple[torch.Tensor, int]:
    """Load from tar:// URI → (audio_tensor [C, T], sample_rate)"""
    tar_path, offset, nbytes = _parse_tar_uri(uri)
    fd = _tar_fd_cache.get_fd(tar_path)  # LRU FD cache (max 256)
    raw = os.pread(fd, nbytes, offset)    # Thread-safe pread
    audio_np, sr = sf.read(BytesIO(raw), dtype="float32")
    # Convert (samples,) or (samples, channels) → torch [C, T]
    return torch.Tensor, sr
```

**Inference Messages Format:**
```python
messages = [
    {"role": "user", "content": "<audio>\nTranscribe the audio."}
]
response = chat_model.chat(
    messages=messages,
    audios=["path/to/audio.wav"]  # or tar:// URIs
)
```

### 4. TRAINING DATA PIPELINE

**Preprocessing Phases (6 total):**

1. **Phase 1 - Audio Benchmarking** (Complete)
   - Benchmark: tar read → FLAC decode → resample 16k mono → ndarray
   - Result: 79.8% of samples needed 48k→16k resampling at runtime
   - Decision: Pre-convert all audio offline

2. **Phase 2 - Audio Conversion** (Complete)
   - Input: 78M samples in 48k/16k mixed sample rates
   - Output: 16kHz mono FLAC in tar files
   - Orchestrated conversion: `phase2_orchestrate_conversion.py` (8 workers)
   - Atomic swap strategy for safety
   - Result: 74.9M train, 1.57M dev, 1.57M test rows

3. **Phase 3 - URI Loader Patch** (Complete)
   - tar:// URI support added to LlamaFactory
   - pread + FD cache for efficient random access
   - Validation: 10k random URI decode, 100% success

4. **Phase 4 - LlamaFactory Dataset Build** (Complete)
   - Input: Phase 2 manifest parquets
   - Output: Alpaca-format parquet datasets
   - Format:
     ```
     instruction: "<audio>\nTranscribe the audio."
     input: "Language: <lang>"
     output: "<transcript>"
     audios: ["tar://path|offset|nbytes"]
     duration_bucket: "0-4s" | "4-8s" | "8-16s" | "16-32s" | ">32s"
     ```
   - Block-shuffle for I/O locality (256-row blocks)
   - All buckets included: dispose, golden, redo, finalsftdata, indicvoices

5. **Phase 5 - Config Generation** (Complete)
   - LoRA smoke test: 4 batch size
   - Full fine-tune: 8 batch size
   - 64 preprocessing workers, 8 dataloader workers
   - disable_shuffling: true (uses block-shuffled order)

6. **Phase 6 - Preflight** (Complete)
   - 10k random URI decode validation
   - Zero SR mismatches enforced
   - All checks passed

### 5. BENCHMARK & EVALUATION SCRIPTS

**Location:** `/home/ubuntu/training/gemma3n-E2B-asr-ft/tools/`

**Key Scripts:**

1. **bench_raw_audio_pipeline.py**
   - Tests: tar → FLAC decode → resample → ndarray
   - Worker sweep: 4, 8, 12, 16, 24, 32 workers
   - Metrics: samples/sec, audio-sec/sec, p50/p95 latency, CPU%
   - Output: JSON + Markdown report

2. **phase3_uri_bench.py**
   - Compares tar:// URI loader vs Phase 2 baseline
   - 2000 stratified samples (by language × duration bucket)
   - Gate: URI perf >= 90% of baseline
   - Result: ALL PASS at all worker counts

3. **phase6_preflight.py**
   - Validates dataset_info.json and parquets
   - 10k random URI decode with 32 workers
   - Checks: no SR mismatches, fallback recovery
   - Saves report: `phase6_preflight.json`

**Existing Artifacts:**
- `/home/ubuntu/training/gemma3n-E2B-asr-ft/artifacts/phase2/` - Global manifest, splits
- `/home/ubuntu/training/gemma3n-E2B-asr-ft/artifacts/phase4/` - LF datasets + dataset_info.json
- `/home/ubuntu/training/gemma3n-E2B-asr-ft/benchmarks/phase1/` - Audio pipeline bench

### 6. TRAINING CONFIGURATION

**Two Main Configs:**

**A. LoRA Smoke Test** (`gemma3n_asr_lora_smoke.yaml`)
```yaml
model_name_or_path: /workspace/gemini-asr/gemma-model
trust_remote_code: true
stage: sft
finetuning_type: lora
lora_rank: 16
lora_alpha: 32
lora_target: self_attn.q_proj,self_attn.k_proj,self_attn.v_proj,self_attn.o_proj,mlp.gate_proj,mlp.up_proj,mlp.down_proj
freeze_multi_modal_projector: false

dataset_dir: /root/gemini-asr/lf_asr/artifacts/phase4
dataset: asr_train
eval_dataset: asr_dev
template: gemma3n

per_device_train_batch_size: 4
per_device_eval_batch_size: 4
gradient_accumulation_steps: 4
learning_rate: 2.0e-4
num_train_epochs: 1.0
max_steps: 2000
warmup_steps: 100

bf16: true
flash_attn: auto
preprocessing_num_workers: 64
dataloader_num_workers: 8
dataloader_persistent_workers: true
dataloader_prefetch_factor: 4

seed: 42
cutoff_len: 2048
```

**B. Full Fine-Tune** (`gemma3n_asr_full_ft.yaml`)
```yaml
finetuning_type: full
freeze_vision_tower: false
per_device_train_batch_size: 8
gradient_accumulation_steps: 8
learning_rate: 5.0e-5
gradient_checkpointing: true
flash_attn: fa2
```

**Data Loading:**
- `disable_shuffling: true` - Uses pre-ordered block-shuffled data
- `remove_unused_columns: false` - Retains metadata columns
- No runtime resample (LLAMAFACTORY_ALLOW_RESAMPLE=0)
- Fallback enabled: 1 second of silence on decode errors

### 7. DEPENDENCIES & ENVIRONMENT

**Key Dependencies** (from imports and configs):
- **PyTorch:** 2.10.0+cu128
- **torchaudio:** 2.10.0+cu128 (for audio processing)
- **transformers:** 5.2.0 (Hugging Face)
- **pandas:** Data manipulation
- **soundfile:** FLAC decoding
- **numpy:** Numerical operations
- **LlamaFactory:** Vendored in `vendor/LlamaFactory/`

**Environment Setup** (from launch_smoke.sh):
```bash
export OMP_NUM_THREADS=1
export MKL_NUM_THREADS=1
export TORCH_NUM_THREADS=1
export LLAMAFACTORY_ALLOW_RESAMPLE=0
export LLAMAFACTORY_AUDIO_DECODE_FALLBACK=1
export LLAMAFACTORY_AUDIO_DECODE_FALLBACK_SEC=1.0
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
ulimit -n 1048576  # File descriptor limit for tar FD cache
```

**GPU:** 8x NVIDIA B200 (183GB each)
**CPU:** 240 cores
**RAM:** 3TB

### 8. WRITING A BENCHMARK SCRIPT - KEY PATTERNS

To write inference/benchmark scripts for this model:

**Pattern 1: Using ChatModel API**
```python
from llamafactory.chat import ChatModel

chat_model = ChatModel({
    "model_name_or_path": "/workspace/gemini-asr/gemma-model",
    "trust_remote_code": True,
    "template": "gemma3n",
})

messages = [{"role": "user", "content": "<audio>\nTranscribe the audio."}]
responses = chat_model.chat(
    messages=messages,
    audios=["tar://path/to/audio.tar|offset|nbytes"],  # or local paths
    temperature=0.0,
    max_new_tokens=256
)
print(responses[0].response_text)
```

**Pattern 2: Direct URI Decoding**
```python
import sys
sys.path.insert(0, "/root/gemini-asr/lf_asr/vendor/LlamaFactory/src")
from llamafactory.data.mm_plugin import _load_audio_from_tar_uri

uri = "tar:///path/audio.tar|1024|5120"
audio_tensor, sr = _load_audio_from_tar_uri(uri)
# audio_tensor: torch.Tensor [C, T]
# sr: sample rate (16000)
```

**Pattern 3: Batch Processing with Manifest**
```python
import pandas as pd
from concurrent.futures import ProcessPoolExecutor

manifest = pd.read_parquet("artifacts/phase4/train.parquet")
uris = manifest["audios"].apply(lambda x: x[0])  # Extract first audio

def transcribe_batch(uri_batch):
    # Use ChatModel.chat() with batch of audios
    pass

with ProcessPoolExecutor(max_workers=8) as executor:
    results = list(executor.map(transcribe_batch, chunks(uris, 32)))
```

**Key Metrics to Track:**
- Throughput: samples/sec, audio-sec/sec
- Latency: p50, p95 (per-sample decoding + inference)
- GPU utilization: torch.cuda.Event for timing
- SR mismatches: enforce 16kHz only
- Decode failures: count and log with fallback recovery

### 9. FILE PATHS REFERENCE

**Key Code Paths:**
- Model loading: `/home/ubuntu/training/gemma3n-E2B-asr-ft/vendor/LlamaFactory/src/llamafactory/model/`
- Chat inference: `/home/ubuntu/training/gemma3n-E2B-asr-ft/vendor/LlamaFactory/src/llamafactory/chat/chat_model.py`
- Audio plugin: `/home/ubuntu/training/gemma3n-E2B-asr-ft/vendor/LlamaFactory/src/llamafactory/data/mm_plugin.py`
- Templates: `/home/ubuntu/training/gemma3n-E2B-asr-ft/vendor/LlamaFactory/src/llamafactory/data/template.py`
- Data processing: `/home/ubuntu/training/gemma3n-E2B-asr-ft/vendor/LlamaFactory/src/llamafactory/data/`

**Data Paths:**
- Train data: `/root/gemini-asr/lf_asr/artifacts/phase4/train.parquet` (74.9M rows)
- Dev data: `/root/gemini-asr/lf_asr/artifacts/phase4/dev.parquet` (1.57M rows)
- Dataset info: `/root/gemini-asr/lf_asr/artifacts/phase4/dataset_info.json`
- Audio shards: Various tar files referenced in manifests

**Checkpoint Output:**
- Smoke training: `/root/gemini-asr/lf_asr/outputs/smoke_run/`
- Full training: `/root/gemini-asr/lf_asr/outputs/full_ft_run/`

### 10. PIPELINE DESIGN PRINCIPLES

1. **Zero Runtime Resampling:** All audio pre-converted offline to 16kHz mono FLAC
2. **Efficient Random Access:** tar:// URIs with pread() + FD cache for O(1) access without extraction
3. **I/O Locality:** Block-shuffling (256-row blocks) keeps adjacent samples from same tar file
4. **Reproducibility:** Fixed seed 42, deterministic splits via MD5(video_id)
5. **Data Integrity:** All metadata logged, drop rates tracked, fallback recovery tested
6. **Multiprocessing Safety:** Thread-safe FD cache, per-process state isolation
7. **No Data Filtering:** All quality buckets (dispose, golden, redo, etc.) included

This is a production-grade ASR fine-tuning pipeline with robust audio handling, efficient data I/O, and comprehensive validation at each phase.