     1→from __future__ import annotations
     2→
     3→import unicodedata
     4→
     5→ISO3_TO_WHISPER_LANG: dict[str, str] = {
     6→    "eng": "en",
     7→    "hin": "hi",
     8→    "mar": "mr",
     9→    "guj": "gu",
    10→    "pan": "pa",
    11→    "tam": "ta",
    12→    "tel": "te",
    13→    "kan": "kn",
    14→    "mal": "ml",
    15→    "ben": "bn",
    16→    "asm": "as",
    17→    "urd": "ur",
    18→    "san": "sa",
    19→    "ori": "or",
    20→    "ory": "or",
    21→}
    22→
    23→# Human-friendly names and scripts (for prompts/UI).
    24→LANG_CODE_TO_NAME: dict[str, str] = {
    25→    "eng": "English",
    26→    "en": "English",
    27→    "hin": "Hindi",
    28→    "hi": "Hindi",
    29→    "mar": "Marathi",
    30→    "mr": "Marathi",
    31→    "guj": "Gujarati",
    32→    "gu": "Gujarati",
    33→    "pan": "Punjabi",
    34→    "pa": "Punjabi",
    35→    "tam": "Tamil",
    36→    "ta": "Tamil",
    37→    "tel": "Telugu",
    38→    "te": "Telugu",
    39→    "kan": "Kannada",
    40→    "kn": "Kannada",
    41→    "mal": "Malayalam",
    42→    "ml": "Malayalam",
    43→    "ben": "Bengali",
    44→    "bn": "Bengali",
    45→    "asm": "Assamese",
    46→    "as": "Assamese",
    47→    "ori": "Odia",
    48→    "ory": "Odia",
    49→    "or": "Odia",
    50→}
    51→
    52→LANG_NAME_TO_SCRIPT: dict[str, str] = {
    53→    "English": "Latin",
    54→    "Hindi": "Devanagari",
    55→    "Marathi": "Devanagari",
    56→    "Gujarati": "Gujarati",
    57→    "Punjabi": "Gurmukhi",
    58→    "Tamil": "Tamil",
    59→    "Telugu": "Telugu",
    60→    "Kannada": "Kannada",
    61→    "Malayalam": "Malayalam",
    62→    "Bengali": "Bengali",
    63→    "Assamese": "Assamese",
    64→    "Odia": "Odia",
    65→}
    66→
    67→# Coarse script expectations for filtering/normalization heuristics.
    68→# Assamese uses the Bengali Unicode block.
    69→LANG_CODE_TO_SCRIPT: dict[str, str] = {
    70→    "en": "Latin",
    71→    "eng": "Latin",
    72→    "hi": "Devanagari",
    73→    "hin": "Devanagari",
    74→    "mr": "Devanagari",
    75→    "mar": "Devanagari",
    76→    "gu": "Gujarati",
    77→    "guj": "Gujarati",
    78→    "pa": "Gurmukhi",
    79→    "pan": "Gurmukhi",
    80→    "ta": "Tamil",
    81→    "tam": "Tamil",
    82→    "te": "Telugu",
    83→    "tel": "Telugu",
    84→    "kn": "Kannada",
    85→    "kan": "Kannada",
    86→    "ml": "Malayalam",
    87→    "mal": "Malayalam",
    88→    "bn": "Bengali",
    89→    "ben": "Bengali",
    90→    "as": "Bengali",
    91→    "asm": "Bengali",
    92→    "or": "Oriya",
    93→    "ori": "Oriya",
    94→    "ory": "Oriya",
    95→    "ur": "Arabic",
    96→    "urd": "Arabic",
    97→}
    98→
    99→
   100→def expected_script(lang_code: str | None) -> str | None:
   101→    if not lang_code:
   102→        return None
   103→    return LANG_CODE_TO_SCRIPT.get(lang_code.strip().lower())
   104→
   105→
   106→def dominant_script(text: str, *, min_letters: int = 4) -> str | None:
   107→    """
   108→    Heuristic: return the dominant script bucket based on Unicode block ranges.
   109→    Returns None if there's not enough signal (e.g., mostly digits/punctuation).
   110→    """
   111→    if not text:
   112→        return None
   113→
   114→    counts: dict[str, int] = {}
   115→    total_letters = 0
   116→
   117→    def bump(script: str) -> None:
   118→        nonlocal total_letters
   119→        counts[script] = counts.get(script, 0) + 1
   120→        total_letters += 1
   121→
   122→    for ch in text:
   123→        if ch.isspace():
   124→            continue
   125→        cat = unicodedata.category(ch)
   126→        if not cat.startswith("L"):
   127→            continue
   128→
   129→        o = ord(ch)
   130→        if 0x0900 <= o <= 0x097F:
   131→            bump("Devanagari")
   132→        elif 0x0980 <= o <= 0x09FF:
   133→            bump("Bengali")
   134→        elif 0x0A00 <= o <= 0x0A7F:
   135→            bump("Gurmukhi")
   136→        elif 0x0A80 <= o <= 0x0AFF:
   137→            bump("Gujarati")
   138→        elif 0x0B00 <= o <= 0x0B7F:
   139→            bump("Oriya")
   140→        elif 0x0B80 <= o <= 0x0BFF:
   141→            bump("Tamil")
   142→        elif 0x0C00 <= o <= 0x0C7F:
   143→            bump("Telugu")
   144→        elif 0x0C80 <= o <= 0x0CFF:
   145→            bump("Kannada")
   146→        elif 0x0D00 <= o <= 0x0D7F:
   147→            bump("Malayalam")
   148→        else:
   149→            # Rough Latin bucket for English/code-switch.
   150→            if ("LATIN" in unicodedata.name(ch, "")) or ("A" <= ch <= "z"):
   151→                bump("Latin")
   152→
   153→    if total_letters < int(min_letters):
   154→        return None
   155→    return max(counts.items(), key=lambda kv: kv[1])[0] if counts else None
   156→
   157→
   158→def to_whisper_lang(lang_code: str | None) -> str | None:
   159→    if not lang_code:
   160→        return None
   161→    lang_code = lang_code.strip().lower()
   162→    if len(lang_code) == 2:
   163→        return lang_code
   164→    return ISO3_TO_WHISPER_LANG.get(lang_code, None)
   165→
   166→
   167→def to_language_name(lang_code: str | None) -> str | None:
   168→    if not lang_code:
   169→        return None
   170→    lang_code = lang_code.strip().lower()
   171→    return LANG_CODE_TO_NAME.get(lang_code)
   172→
   173→
   174→def to_prompt_language_hint(lang_code: str | None) -> str:
   175→    """
   176→    Return a human-friendly language hint string for prompting audio-native models.
   177→    """
   178→    if not lang_code:
   179→        return "Unknown (auto-detect primary spoken language)"
   180→    lang_code_norm = lang_code.strip().lower()
   181→    name = LANG_CODE_TO_NAME.get(lang_code_norm)
   182→    if not name:
   183→        return f"{lang_code_norm} (auto-detect primary spoken language)"
   184→    script = LANG_NAME_TO_SCRIPT.get(name)
   185→    if script:
   186→        return f"{name} ({script} script)"
   187→    return name
   188→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

from __future__ import annotations

import unicodedata

ISO3_TO_WHISPER_LANG: dict[str, str] = {
    "eng": "en",
    "hin": "hi",
    "mar": "mr",
    "guj": "gu",
    "pan": "pa",
    "tam": "ta",
    "tel": "te",
    "kan": "kn",
    "mal": "ml",
    "ben": "bn",
    "asm": "as",
    "urd": "ur",
    "san": "sa",
    "ori": "or",
    "ory": "or",
}

# Human-friendly names and scripts (for prompts/UI).
LANG_CODE_TO_NAME: dict[str, str] = {
    "eng": "English",
    "en": "English",
    "hin": "Hindi",
    "hi": "Hindi",
    "mar": "Marathi",
    "mr": "Marathi",
    "guj": "Gujarati",
    "gu": "Gujarati",
    "pan": "Punjabi",
    "pa": "Punjabi",
    "tam": "Tamil",
    "ta": "Tamil",
    "tel": "Telugu",
    "te": "Telugu",
    "kan": "Kannada",
    "kn": "Kannada",
    "mal": "Malayalam",
    "ml": "Malayalam",
    "ben": "Bengali",
    "bn": "Bengali",
    "asm": "Assamese",
    "as": "Assamese",
    "ori": "Odia",
    "ory": "Odia",
    "or": "Odia",
}

LANG_NAME_TO_SCRIPT: dict[str, str] = {
    "English": "Latin",
    "Hindi": "Devanagari",
    "Marathi": "Devanagari",
    "Gujarati": "Gujarati",
    "Punjabi": "Gurmukhi",
    "Tamil": "Tamil",
    "Telugu": "Telugu",
    "Kannada": "Kannada",
    "Malayalam": "Malayalam",
    "Bengali": "Bengali",
    "Assamese": "Assamese",
    "Odia": "Odia",
}

# Coarse script expectations for filtering/normalization heuristics.
# Assamese uses the Bengali Unicode block.
LANG_CODE_TO_SCRIPT: dict[str, str] = {
    "en": "Latin",
    "eng": "Latin",
    "hi": "Devanagari",
    "hin": "Devanagari",
    "mr": "Devanagari",
    "mar": "Devanagari",
    "gu": "Gujarati",
    "guj": "Gujarati",
    "pa": "Gurmukhi",
    "pan": "Gurmukhi",
    "ta": "Tamil",
    "tam": "Tamil",
    "te": "Telugu",
    "tel": "Telugu",
    "kn": "Kannada",
    "kan": "Kannada",
    "ml": "Malayalam",
    "mal": "Malayalam",
    "bn": "Bengali",
    "ben": "Bengali",
    "as": "Bengali",
    "asm": "Bengali",
    "or": "Oriya",
    "ori": "Oriya",
    "ory": "Oriya",
    "ur": "Arabic",
    "urd": "Arabic",
}

def expected_script(lang_code: str | None) -> str | None:
    if not lang_code:
        return None
    return LANG_CODE_TO_SCRIPT.get(lang_code.strip().lower())

def dominant_script(text: str, *, min_letters: int = 4) -> str | None:
    """
    Heuristic: return the dominant script bucket based on Unicode block ranges.
    Returns None if there's not enough signal (e.g., mostly digits/punctuation).
    """
    if not text:
        return None

    counts: dict[str, int] = {}
    total_letters = 0

    def bump(script: str) -> None:
        nonlocal total_letters
        counts[script] = counts.get(script, 0) + 1
        total_letters += 1

    for ch in text:
        if ch.isspace():
            continue
        cat = unicodedata.category(ch)
        if not cat.startswith("L"):
            continue

        o = ord(ch)
        if 0x0900 <= o <= 0x097F:
            bump("Devanagari")
        elif 0x0980 <= o <= 0x09FF:
            bump("Bengali")
        elif 0x0A00 <= o <= 0x0A7F:
            bump("Gurmukhi")
        elif 0x0A80 <= o <= 0x0AFF:
            bump("Gujarati")
        elif 0x0B00 <= o <= 0x0B7F:
            bump("Oriya")
        elif 0x0B80 <= o <= 0x0BFF:
            bump("Tamil")
        elif 0x0C00 <= o <= 0x0C7F:
            bump("Telugu")
        elif 0x0C80 <= o <= 0x0CFF:
            bump("Kannada")
        elif 0x0D00 <= o <= 0x0D7F:
            bump("Malayalam")
        else:
            # Rough Latin bucket for English/code-switch.
            if ("LATIN" in unicodedata.name(ch, "")) or ("A" <= ch <= "z"):
                bump("Latin")

    if total_letters < int(min_letters):
        return None
    return max(counts.items(), key=lambda kv: kv[1])[0] if counts else None

def to_whisper_lang(lang_code: str | None) -> str | None:
    if not lang_code:
        return None
    lang_code = lang_code.strip().lower()
    if len(lang_code) == 2:
        return lang_code
    return ISO3_TO_WHISPER_LANG.get(lang_code, None)

def to_language_name(lang_code: str | None) -> str | None:
    if not lang_code:
        return None
    lang_code = lang_code.strip().lower()
    return LANG_CODE_TO_NAME.get(lang_code)

def to_prompt_language_hint(lang_code: str | None) -> str:
    """
    Return a human-friendly language hint string for prompting audio-native models.
    """
    if not lang_code:
        return "Unknown (auto-detect primary spoken language)"
    lang_code_norm = lang_code.strip().lower()
    name = LANG_CODE_TO_NAME.get(lang_code_norm)
    if not name:
        return f"{lang_code_norm} (auto-detect primary spoken language)"
    script = LANG_NAME_TO_SCRIPT.get(name)
    if script:
        return f"{name} ({script} script)"
    return name


     1→from __future__ import annotations
     2→
     3→from dataclasses import dataclass
     4→from pathlib import Path
     5→from typing import Final
     6→
     7→from dotenv import load_dotenv
     8→
     9→
    10→@dataclass(frozen=True)
    11→class Settings:
    12→    r2_endpoint_url: str
    13→    r2_access_key_id: str
    14→    r2_secret_access_key: str
    15→    r2_bucket: str = "1-cleaned-data"
    16→    r2_prefix: str | None = None
    17→
    18→    @staticmethod
    19→    def load(env_path: str | Path = ".env") -> "Settings":
    20→        load_dotenv(dotenv_path=env_path, override=False)
    21→
    22→        import os
    23→
    24→        def required(name: str) -> str:
    25→            value = os.getenv(name)
    26→            if not value:
    27→                raise RuntimeError(f"Missing required env var: {name}")
    28→            return value
    29→
    30→        endpoint_url: Final[str] = required("R2_ENDPOINT_URL")
    31→        access_key_id: Final[str] = required("R2_ACCESS_KEY_ID")
    32→        secret_access_key: Final[str] = required("R2_SECRET_ACCESS_KEY")
    33→        bucket: Final[str] = os.getenv("R2_BUCKET") or "1-cleaned-data"
    34→        prefix = os.getenv("R2_PREFIX") or None
    35→
    36→        return Settings(
    37→            r2_endpoint_url=endpoint_url,
    38→            r2_access_key_id=access_key_id,
    39→            r2_secret_access_key=secret_access_key,
    40→            r2_bucket=bucket,
    41→            r2_prefix=prefix,
    42→        )
    43→
    44→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Final

from dotenv import load_dotenv

@dataclass(frozen=True)
class Settings:
    r2_endpoint_url: str
    r2_access_key_id: str
    r2_secret_access_key: str
    r2_bucket: str = "1-cleaned-data"
    r2_prefix: str | None = None

    @staticmethod
    def load(env_path: str | Path = ".env") -> "Settings":
        load_dotenv(dotenv_path=env_path, override=False)

        import os

        def required(name: str) -> str:
            value = os.getenv(name)
            if not value:
                raise RuntimeError(f"Missing required env var: {name}")
            return value

        endpoint_url: Final[str] = required("R2_ENDPOINT_URL")
        access_key_id: Final[str] = required("R2_ACCESS_KEY_ID")
        secret_access_key: Final[str] = required("R2_SECRET_ACCESS_KEY")
        bucket: Final[str] = os.getenv("R2_BUCKET") or "1-cleaned-data"
        prefix = os.getenv("R2_PREFIX") or None

        return Settings(
            r2_endpoint_url=endpoint_url,
            r2_access_key_id=access_key_id,
            r2_secret_access_key=secret_access_key,
            r2_bucket=bucket,
            r2_prefix=prefix,
        )


Now let me examine more supporting files:


     1→from __future__ import annotations
     2→
     3→import json
     4→import subprocess
     5→from dataclasses import dataclass
     6→from pathlib import Path
     7→
     8→import numpy as np
     9→
    10→
    11→@dataclass(frozen=True)
    12→class AudioInfo:
    13→    path: str
    14→    duration_sec: float | None
    15→    sample_rate: int | None
    16→    channels: int | None
    17→
    18→
    19→@dataclass(frozen=True)
    20→class AudioQCMetrics:
    21→    duration_sec: float
    22→    rms_dbfs: float
    23→    peak_dbfs: float
    24→    clipping_fraction: float
    25→
    26→
    27→def ffprobe(path: Path) -> AudioInfo:
    28→    cmd = [
    29→        "ffprobe",
    30→        "-v",
    31→        "error",
    32→        "-show_entries",
    33→        "format=duration",
    34→        "-show_entries",
    35→        "stream=sample_rate,channels",
    36→        "-of",
    37→        "json",
    38→        str(path),
    39→    ]
    40→    proc = subprocess.run(cmd, check=True, capture_output=True, text=True)
    41→    data = json.loads(proc.stdout or "{}")
    42→
    43→    duration_sec = None
    44→    try:
    45→        duration_sec = float(data.get("format", {}).get("duration"))
    46→    except (TypeError, ValueError):
    47→        duration_sec = None
    48→
    49→    sample_rate = None
    50→    channels = None
    51→    streams = data.get("streams") or []
    52→    if streams:
    53→        s0 = streams[0] or {}
    54→        try:
    55→            sample_rate = int(s0.get("sample_rate")) if s0.get("sample_rate") else None
    56→        except (TypeError, ValueError):
    57→            sample_rate = None
    58→        try:
    59→            channels = int(s0.get("channels")) if s0.get("channels") else None
    60→        except (TypeError, ValueError):
    61→            channels = None
    62→
    63→    return AudioInfo(path=str(path), duration_sec=duration_sec, sample_rate=sample_rate, channels=channels)
    64→
    65→
    66→def decode_pcm16(path: Path, sample_rate: int = 16000, mono: bool = True) -> tuple[np.ndarray, int]:
    67→    """
    68→    Decode audio with ffmpeg into signed 16-bit PCM (optionally mono) at `sample_rate`.
    69→    Returns (float32 waveform in [-1, 1], sample_rate).
    70→    """
    71→    cmd = ["ffmpeg", "-v", "error", "-i", str(path), "-f", "s16le", "-ar", str(sample_rate)]
    72→    if mono:
    73→        cmd += ["-ac", "1"]
    74→    cmd += ["-"]
    75→
    76→    proc = subprocess.run(cmd, check=True, capture_output=True)
    77→    pcm = np.frombuffer(proc.stdout, dtype=np.int16)
    78→    if pcm.size == 0:
    79→        return np.zeros((0,), dtype=np.float32), sample_rate
    80→    wav = pcm.astype(np.float32) / 32768.0
    81→    return wav, sample_rate
    82→
    83→
    84→def compute_qc(path: Path, sample_rate: int = 16000) -> AudioQCMetrics:
    85→    wav, sr = decode_pcm16(path, sample_rate=sample_rate, mono=True)
    86→    duration_sec = float(wav.shape[0]) / float(sr) if sr else 0.0
    87→    if wav.size == 0:
    88→        return AudioQCMetrics(duration_sec=0.0, rms_dbfs=float("-inf"), peak_dbfs=float("-inf"), clipping_fraction=0.0)
    89→
    90→    peak = float(np.max(np.abs(wav)))
    91→    rms = float(np.sqrt(np.mean(np.square(wav))))
    92→
    93→    peak_dbfs = 20.0 * float(np.log10(max(peak, 1e-12)))
    94→    rms_dbfs = 20.0 * float(np.log10(max(rms, 1e-12)))
    95→
    96→    clipping_threshold = 0.999
    97→    clipping_fraction = float(np.mean(np.abs(wav) >= clipping_threshold))
    98→
    99→    return AudioQCMetrics(
   100→        duration_sec=duration_sec,
   101→        rms_dbfs=rms_dbfs,
   102→        peak_dbfs=peak_dbfs,
   103→        clipping_fraction=clipping_fraction,
   104→    )
   105→
   106→
   107→def normalize_audio(
   108→    in_path: Path,
   109→    out_path: Path,
   110→    sample_rate: int = 16000,
   111→    mono: bool = True,
   112→    trim_silence: bool = True,
   113→    silence_threshold_db: float = -40.0,
   114→    min_silence_sec: float = 0.15,
   115→) -> Path:
   116→    """
   117→    Convert audio to 16kHz mono WAV and optionally trim leading/trailing silence.
   118→    """
   119→    out_path.parent.mkdir(parents=True, exist_ok=True)
   120→
   121→    cmd = ["ffmpeg", "-v", "error", "-y", "-i", str(in_path)]
   122→
   123→    if trim_silence:
   124→        # Trim leading/trailing silence; tuned for already-clean segments.
   125→        # https://ffmpeg.org/ffmpeg-filters.html#silenceremove
   126→        thr = f"{silence_threshold_db}dB"
   127→        sil = (
   128→            f"silenceremove=start_periods=1:start_duration={min_silence_sec}:start_threshold={thr}:"
   129→            f"stop_periods=1:stop_duration={min_silence_sec}:stop_threshold={thr}"
   130→        )
   131→        cmd += ["-af", sil]
   132→
   133→    cmd += ["-ar", str(sample_rate)]
   134→    if mono:
   135→        cmd += ["-ac", "1"]
   136→    cmd += ["-c:a", "pcm_s16le", str(out_path)]
   137→
   138→    subprocess.run(cmd, check=True)
   139→    return out_path
   140→
   141→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

from __future__ import annotations

import json
import subprocess
from dataclasses import dataclass
from pathlib import Path

import numpy as np

@dataclass(frozen=True)
class AudioInfo:
    path: str
    duration_sec: float | None
    sample_rate: int | None
    channels: int | None

@dataclass(frozen=True)
class AudioQCMetrics:
    duration_sec: float
    rms_dbfs: float
    peak_dbfs: float
    clipping_fraction: float

def ffprobe(path: Path) -> AudioInfo:
    cmd = [
        "ffprobe",
        "-v",
        "error",
        "-show_entries",
        "format=duration",
        "-show_entries",
        "stream=sample_rate,channels",
        "-of",
        "json",
        str(path),
    ]
    proc = subprocess.run(cmd, check=True, capture_output=True, text=True)
    data = json.loads(proc.stdout or "{}")

    duration_sec = None
    try:
        duration_sec = float(data.get("format", {}).get("duration"))
    except (TypeError, ValueError):
        duration_sec = None

    sample_rate = None
    channels = None
    streams = data.get("streams") or []
    if streams:
        s0 = streams[0] or {}
        try:
            sample_rate = int(s0.get("sample_rate")) if s0.get("sample_rate") else None
        except (TypeError, ValueError):
            sample_rate = None
        try:
            channels = int(s0.get("channels")) if s0.get("channels") else None
        except (TypeError, ValueError):
            channels = None

    return AudioInfo(path=str(path), duration_sec=duration_sec, sample_rate=sample_rate, channels=channels)

def decode_pcm16(path: Path, sample_rate: int = 16000, mono: bool = True) -> tuple[np.ndarray, int]:
    """
    Decode audio with ffmpeg into signed 16-bit PCM (optionally mono) at `sample_rate`.
    Returns (float32 waveform in [-1, 1], sample_rate).
    """
    cmd = ["ffmpeg", "-v", "error", "-i", str(path), "-f", "s16le", "-ar", str(sample_rate)]
    if mono:
        cmd += ["-ac", "1"]
    cmd += ["-"]

    proc = subprocess.run(cmd, check=True, capture_output=True)
    pcm = np.frombuffer(proc.stdout, dtype=np.int16)
    if pcm.size == 0:
        return np.zeros((0,), dtype=np.float32), sample_rate
    wav = pcm.astype(np.float32) / 32768.0
    return wav, sample_rate

def compute_qc(path: Path, sample_rate: int = 16000) -> AudioQCMetrics:
    wav, sr = decode_pcm16(path, sample_rate=sample_rate, mono=True)
    duration_sec = float(wav.shape[0]) / float(sr) if sr else 0.0
    if wav.size == 0:
        return AudioQCMetrics(duration_sec=0.0, rms_dbfs=float("-inf"), peak_dbfs=float("-inf"), clipping_fraction=0.0)

    peak = float(np.max(np.abs(wav)))
    rms = float(np.sqrt(np.mean(np.square(wav))))

    peak_dbfs = 20.0 * float(np.log10(max(peak, 1e-12)))
    rms_dbfs = 20.0 * float(np.log10(max(rms, 1e-12)))

    clipping_threshold = 0.999
    clipping_fraction = float(np.mean(np.abs(wav) >= clipping_threshold))

    return AudioQCMetrics(
        duration_sec=duration_sec,
        rms_dbfs=rms_dbfs,
        peak_dbfs=peak_dbfs,
        clipping_fraction=clipping_fraction,
    )

def normalize_audio(
    in_path: Path,
    out_path: Path,
    sample_rate: int = 16000,
    mono: bool = True,
    trim_silence: bool = True,
    silence_threshold_db: float = -40.0,
    min_silence_sec: float = 0.15,
) -> Path:
    """
    Convert audio to 16kHz mono WAV and optionally trim leading/trailing silence.
    """
    out_path.parent.mkdir(parents=True, exist_ok=True)

    cmd = ["ffmpeg", "-v", "error", "-y", "-i", str(in_path)]

    if trim_silence:
        # Trim leading/trailing silence; tuned for already-clean segments.
        # https://ffmpeg.org/ffmpeg-filters.html#silenceremove
        thr = f"{silence_threshold_db}dB"
        sil = (
            f"silenceremove=start_periods=1:start_duration={min_silence_sec}:start_threshold={thr}:"
            f"stop_periods=1:stop_duration={min_silence_sec}:stop_threshold={thr}"
        )
        cmd += ["-af", sil]

    cmd += ["-ar", str(sample_rate)]
    if mono:
        cmd += ["-ac", "1"]
    cmd += ["-c:a", "pcm_s16le", str(out_path)]

    subprocess.run(cmd, check=True)
    return out_path


     1→from __future__ import annotations
     2→
     3→from dataclasses import dataclass
     4→
     5→
     6→@dataclass(frozen=True)
     7→class AlignmentResult:
     8→    distance: int
     9→    aligned: list[tuple[str | None, str | None]]
    10→
    11→
    12→def align_tokens(ref: list[str], hyp: list[str]) -> AlignmentResult:
    13→    """
    14→    Levenshtein alignment between token sequences.
    15→    Returns edit distance and an alignment as (ref_token|None, hyp_token|None) pairs.
    16→    """
    17→    n = len(ref)
    18→    m = len(hyp)
    19→    dp = [[0] * (m + 1) for _ in range(n + 1)]
    20→
    21→    for i in range(1, n + 1):
    22→        dp[i][0] = i
    23→    for j in range(1, m + 1):
    24→        dp[0][j] = j
    25→
    26→    for i in range(1, n + 1):
    27→        ri = ref[i - 1]
    28→        for j in range(1, m + 1):
    29→            hj = hyp[j - 1]
    30→            sub_cost = 0 if ri == hj else 1
    31→            dp[i][j] = min(
    32→                dp[i - 1][j] + 1,  # deletion
    33→                dp[i][j - 1] + 1,  # insertion
    34→                dp[i - 1][j - 1] + sub_cost,  # substitution/match
    35→            )
    36→
    37→    aligned: list[tuple[str | None, str | None]] = []
    38→    i, j = n, m
    39→    while i > 0 or j > 0:
    40→        if i > 0 and j > 0:
    41→            ri = ref[i - 1]
    42→            hj = hyp[j - 1]
    43→            sub_cost = 0 if ri == hj else 1
    44→            if dp[i][j] == dp[i - 1][j - 1] + sub_cost:
    45→                aligned.append((ri, hj))
    46→                i -= 1
    47→                j -= 1
    48→                continue
    49→        if i > 0 and dp[i][j] == dp[i - 1][j] + 1:
    50→            aligned.append((ref[i - 1], None))
    51→            i -= 1
    52→            continue
    53→        if j > 0 and dp[i][j] == dp[i][j - 1] + 1:
    54→            aligned.append((None, hyp[j - 1]))
    55→            j -= 1
    56→            continue
    57→
    58→        # Fallback (shouldn't happen): prefer diagonal, then deletion, then insertion.
    59→        if i > 0 and j > 0:
    60→            aligned.append((ref[i - 1], hyp[j - 1]))
    61→            i -= 1
    62→            j -= 1
    63→        elif i > 0:
    64→            aligned.append((ref[i - 1], None))
    65→            i -= 1
    66→        else:
    67→            aligned.append((None, hyp[j - 1]))
    68→            j -= 1
    69→
    70→    aligned.reverse()
    71→    return AlignmentResult(distance=dp[n][m], aligned=aligned)
    72→
    73→
    74→def edit_distance(a: list[str], b: list[str]) -> int:
    75→    return align_tokens(a, b).distance
    76→
    77→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

from __future__ import annotations

from dataclasses import dataclass

@dataclass(frozen=True)
class AlignmentResult:
    distance: int
    aligned: list[tuple[str | None, str | None]]

def align_tokens(ref: list[str], hyp: list[str]) -> AlignmentResult:
    """
    Levenshtein alignment between token sequences.
    Returns edit distance and an alignment as (ref_token|None, hyp_token|None) pairs.
    """
    n = len(ref)
    m = len(hyp)
    dp = [[0] * (m + 1) for _ in range(n + 1)]

    for i in range(1, n + 1):
        dp[i][0] = i
    for j in range(1, m + 1):
        dp[0][j] = j

    for i in range(1, n + 1):
        ri = ref[i - 1]
        for j in range(1, m + 1):
            hj = hyp[j - 1]
            sub_cost = 0 if ri == hj else 1
            dp[i][j] = min(
                dp[i - 1][j] + 1,  # deletion
                dp[i][j - 1] + 1,  # insertion
                dp[i - 1][j - 1] + sub_cost,  # substitution/match
            )

    aligned: list[tuple[str | None, str | None]] = []
    i, j = n, m
    while i > 0 or j > 0:
        if i > 0 and j > 0:
            ri = ref[i - 1]
            hj = hyp[j - 1]
            sub_cost = 0 if ri == hj else 1
            if dp[i][j] == dp[i - 1][j - 1] + sub_cost:
                aligned.append((ri, hj))
                i -= 1
                j -= 1
                continue
        if i > 0 and dp[i][j] == dp[i - 1][j] + 1:
            aligned.append((ref[i - 1], None))
            i -= 1
            continue
        if j > 0 and dp[i][j] == dp[i][j - 1] + 1:
            aligned.append((None, hyp[j - 1]))
            j -= 1
            continue

        # Fallback (shouldn't happen): prefer diagonal, then deletion, then insertion.
        if i > 0 and j > 0:
            aligned.append((ref[i - 1], hyp[j - 1]))
            i -= 1
            j -= 1
        elif i > 0:
            aligned.append((ref[i - 1], None))
            i -= 1
        else:
            aligned.append((None, hyp[j - 1]))
            j -= 1

    aligned.reverse()
    return AlignmentResult(distance=dp[n][m], aligned=aligned)

def edit_distance(a: list[str], b: list[str]) -> int:
    return align_tokens(a, b).distance


     1→from __future__ import annotations
     2→
     3→import os
     4→import time
     5→import unicodedata
     6→from dataclasses import dataclass
     7→from pathlib import Path
     8→from typing import Any
     9→
    10→from maya_transcribe.asr import ASRResult
    11→from maya_transcribe.language import to_prompt_language_hint
    12→from maya_transcribe.textnorm import normalize_text
    13→
    14→
    15→def _get_api_key() -> str:
    16→    # Support common env var names; `.env` uses GEMINI_KEY in this repo.
    17→    key = (
    18→        os.getenv("GEMINI_KEY")
    19→        or os.getenv("GEMINI_API_KEY")
    20→        or os.getenv("GOOGLE_API_KEY")
    21→        or os.getenv("GOOGLE_GENAI_API_KEY")
    22→    )
    23→    if not key:
    24→        raise RuntimeError("Missing Gemini API key (set GEMINI_KEY or GOOGLE_API_KEY)")
    25→    return key
    26→
    27→
    28→def _mime_type_for_path(path: Path) -> str:
    29→    ext = path.suffix.lower().lstrip(".")
    30→    if ext == "wav":
    31→        return "audio/wav"
    32→    if ext == "mp3":
    33→        return "audio/mpeg"
    34→    if ext == "flac":
    35→        return "audio/flac"
    36→    if ext in {"m4a", "mp4"}:
    37→        return "audio/mp4"
    38→    if ext in {"ogg", "opus"}:
    39→        return "audio/ogg"
    40→    return "application/octet-stream"
    41→
    42→
    43→def load_prompt_template(prompt_path: Path | None = None) -> str:
    44→    """
    45→    Load the prompt text from Prompt.md (preferred) or fall back to a built-in template.
    46→    """
    47→    if prompt_path and prompt_path.exists():
    48→        text = prompt_path.read_text(encoding="utf-8")
    49→        start = text.find("`Instruction:")
    50→        if start != -1:
    51→            start += 1  # drop the leading backtick
    52→            end = text.rfind("`")
    53→            if end != -1 and end > start:
    54→                return text[start:end].strip()
    55→            return text[start:].strip()
    56→        return text.strip()
    57→
    58→    return (
    59→        "Instruction: You are a strict, verbatim transcription engine.\n"
    60→        "Input: Audio file attached\n"
    61→        "Language hint (primary): <language>\n\n"
    62→        "Task:\n"
    63→        "- Transcribe only what is clearly spoken in the audio.\n"
    64→        "- Do NOT guess or “fill in” missing words. If unclear, output <unk>.\n"
    65→        "- Preserve repetitions, filler words, stutters, and false starts.\n"
    66→        "- Do NOT correct grammar, spelling, or pronunciation.\n"
    67→        "- Do NOT translate or summarize.\n"
    68→        "- Do NOT add punctuation.\n\n"
    69→        "Code-switching:\n"
    70→        "- Use the native script for the primary language.\n"
    71→        "- If English words are spoken, keep them in Latin script as spoken.\n\n"
    72→        "OUTPUT FORMAT (exact):\n"
    73→        "Output 1: Transcript\n"
    74→        "<single transcript line>\n"
    75→        "Do not output anything else.\n"
    76→    ).strip()
    77→
    78→@dataclass(frozen=True)
    79→class ParsedGeminiOutput:
    80→    transcript: str
    81→    had_label: bool
    82→    joined_lines: bool
    83→    stripped_punctuation: int
    84→    extra_outputs_present: bool
    85→
    86→
    87→def _strip_punctuation_to_space(text: str) -> tuple[str, int]:
    88→    out: list[str] = []
    89→    removed = 0
    90→    for ch in text:
    91→        if unicodedata.category(ch).startswith("P"):
    92→            out.append(" ")
    93→            removed += 1
    94→        else:
    95→            out.append(ch)
    96→    cleaned = " ".join("".join(out).split())
    97→    return cleaned, removed
    98→
    99→
   100→def parse_strict_output(text: str) -> ParsedGeminiOutput:
   101→    """
   102→    Extract the transcript from the strict output format.
   103→
   104→    Production note: even with strict prompts, some models may add punctuation or line breaks.
   105→    We salvage by joining lines and stripping punctuation (since the pipeline normalizes later anyway).
   106→    """
   107→    text = (text or "").replace("\r\n", "\n").replace("\r", "\n").strip()
   108→    if not text:
   109→        raise ValueError("empty_response")
   110→
   111→    raw_lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
   112→    lines = [ln for ln in raw_lines if ln != "```" and not ln.startswith("```")]
   113→    if not lines:
   114→        raise ValueError("empty_response")
   115→
   116→    lowered = "\n".join(lines).lower()
   117→    extra_outputs_present = ("output 2" in lowered) or ("output 3" in lowered) or ("output 4" in lowered)
   118→
   119→    had_label = False
   120→    transcript_lines: list[str] = []

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

from __future__ import annotations

import os
import time
import unicodedata
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from maya_transcribe.asr import ASRResult
from maya_transcribe.language import to_prompt_language_hint
from maya_transcribe.textnorm import normalize_text

def _get_api_key() -> str:
    # Support common env var names; `.env` uses GEMINI_KEY in this repo.
    key = (
        os.getenv("GEMINI_KEY")
        or os.getenv("GEMINI_API_KEY")
        or os.getenv("GOOGLE_API_KEY")
        or os.getenv("GOOGLE_GENAI_API_KEY")
    )
    if not key:
        raise RuntimeError("Missing Gemini API key (set GEMINI_KEY or GOOGLE_API_KEY)")
    return key

def _mime_type_for_path(path: Path) -> str:
    ext = path.suffix.lower().lstrip(".")
    if ext == "wav":
        return "audio/wav"
    if ext == "mp3":
        return "audio/mpeg"
    if ext == "flac":
        return "audio/flac"
    if ext in {"m4a", "mp4"}:
        return "audio/mp4"
    if ext in {"ogg", "opus"}:
        return "audio/ogg"
    return "application/octet-stream"

def load_prompt_template(prompt_path: Path | None = None) -> str:
    """
    Load the prompt text from Prompt.md (preferred) or fall back to a built-in template.
    """
    if prompt_path and prompt_path.exists():
        text = prompt_path.read_text(encoding="utf-8")
        start = text.find("`Instruction:")
        if start != -1:
            start += 1  # drop the leading backtick
            end = text.rfind("`")
            if end != -1 and end > start:
                return text[start:end].strip()
            return text[start:].strip()
        return text.strip()

    return (
        "Instruction: You are a strict, verbatim transcription engine.\n"
        "Input: Audio file attached\n"
        "Language hint (primary): <language>\n\n"
        "Task:\n"
        "- Transcribe only what is clearly spoken in the audio.\n"
        "- Do NOT guess or “fill in” missing words. If unclear, output <unk>.\n"
        "- Preserve repetitions, filler words, stutters, and false starts.\n"
        "- Do NOT correct grammar, spelling, or pronunciation.\n"
        "- Do NOT translate or summarize.\n"
        "- Do NOT add punctuation.\n\n"
        "Code-switching:\n"
        "- Use the native script for the primary language.\n"
        "- If English words are spoken, keep them in Latin script as spoken.\n\n"
        "OUTPUT FORMAT (exact):\n"
        "Output 1: Transcript\n"
        "<single transcript line>\n"
        "Do not output anything else.\n"
    ).strip()

@dataclass(frozen=True)
class ParsedGeminiOutput:
    transcript: str
    had_label: bool
    joined_lines: bool
    stripped_punctuation: int
    extra_outputs_present: bool

def _strip_punctuation_to_space(text: str) -> tuple[str, int]:
    out: list[str] = []
    removed = 0
    for ch in text:
        if unicodedata.category(ch).startswith("P"):
            out.append(" ")
            removed += 1
        else:
            out.append(ch)
    cleaned = " ".join("".join(out).split())
    return cleaned, removed

def parse_strict_output(text: str) -> ParsedGeminiOutput:
    """
    Extract the transcript from the strict output format.

    Production note: even with strict prompts, some models may add punctuation or line breaks.
    We salvage by joining lines and stripping punctuation (since the pipeline normalizes later anyway).
    """
    text = (text or "").replace("\r\n", "\n").replace("\r", "\n").strip()
    if not text:
        raise ValueError("empty_response")

    raw_lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
    lines = [ln for ln in raw_lines if ln != "```" and not ln.startswith("```")]
    if not lines:
        raise ValueError("empty_response")

    lowered = "\n".join(lines).lower()
    extra_outputs_present = ("output 2" in lowered) or ("output 3" in lowered) or ("output 4" in lowered)

    had_label = False
    transcript_lines: list[str] = []


     1→from __future__ import annotations
     2→
     3→import json
     4→from pathlib import Path
     5→
     6→import typer
     7→from dotenv import load_dotenv
     8→
     9→from maya_transcribe.audio import compute_qc, ffprobe, normalize_audio
    10→from maya_transcribe.config import Settings
    11→from maya_transcribe.dataset import find_segments_dir
    12→from maya_transcribe.ids import segment_identity
    13→from maya_transcribe.io import write_jsonl
    14→from maya_transcribe.r2 import R2Client
    15→from maya_transcribe.tar import safe_extract_tar
    16→
    17→load_dotenv(dotenv_path=Path(".env"), override=False)
    18→
    19→app = typer.Typer(add_completion=False, no_args_is_help=True)
    20→bench_app = typer.Typer(add_completion=False, no_args_is_help=True, help="Benchmarking utilities.")
    21→app.add_typer(bench_app, name="bench")
    22→
    23→
    24→@app.command()
    25→def download(
    26→    video_id: str = typer.Option(..., help="YouTube video id (used to locate the tar in R2)."),
    27→    out: Path = typer.Option(Path("data"), help="Output directory."),
    28→    r2_key: str | None = typer.Option(None, help="Exact R2 object key (skips search)."),
    29→    bucket: str | None = typer.Option(None, help="R2 bucket override (defaults to env R2_BUCKET)."),
    30→    prefix: str | None = typer.Option(None, help="Optional R2 prefix override."),
    31→    use_db: bool = typer.Option(False, help="Fetch `r2_tar_key` from Supabase Postgres (DATABASE_URL)."),
    32→    dry_run: bool = typer.Option(False, help="Only show matched key(s); do not download."),
    33→) -> None:
    34→    """
    35→    Download the cleaned/segmented tar for a video from R2 and extract it.
    36→    """
    37→    settings = Settings.load()
    38→    r2 = R2Client.from_settings(settings)
    39→    if bucket:
    40→        r2 = R2Client(
    41→            endpoint_url=r2.endpoint_url,
    42→            access_key_id=r2.access_key_id,
    43→            secret_access_key=r2.secret_access_key,
    44→            bucket=bucket,
    45→        )
    46→
    47→    out.mkdir(parents=True, exist_ok=True)
    48→    r2_prefix = prefix if prefix is not None else settings.r2_prefix
    49→
    50→    matches: list[str]
    51→    if use_db and not r2_key:
    52→        from maya_transcribe.db import fetch_video_meta
    53→
    54→        meta = fetch_video_meta(video_id)
    55→        if meta and meta.r2_tar_key:
    56→            r2_key = meta.r2_tar_key
    57→
    58→    if r2_key:
    59→        matches = [r2_key]
    60→    else:
    61→        best = r2.find_video_tar_key(video_id=video_id, prefix=r2_prefix)
    62→        if not best:
    63→            typer.echo("No matching objects found in R2.")
    64→            raise typer.Exit(code=2)
    65→        matches = [best]
    66→
    67→    if dry_run:
    68→        for key in matches:
    69→            typer.echo(key)
    70→        return
    71→
    72→    key = matches[0]
    73→    tar_path = r2.download_to(key=key, out_dir=out / video_id)
    74→    extract_dir = out / video_id
    75→    safe_extract_tar(tar_path, extract_dir=extract_dir)
    76→    segments_dir = find_segments_dir(extract_dir)
    77→    typer.echo(f"Downloaded: {tar_path}")
    78→    typer.echo(f"Extracted to: {extract_dir}")
    79→    if segments_dir:
    80→        typer.echo(f"Segments dir: {segments_dir}")
    81→
    82→
    83→@app.command()
    84→def preprocess(
    85→    segments_dir: Path = typer.Option(..., exists=True, file_okay=False, help="Directory containing audio segments."),
    86→    out: Path = typer.Option(Path("outputs/preprocessed"), help="Output directory for normalized audio + QC."),
    87→    sample_rate: int = typer.Option(16000, help="Target sample rate."),
    88→    trim_silence: bool = typer.Option(True, help="Trim leading/trailing silence (ffmpeg silenceremove)."),
    89→    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
    90→    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
    91→) -> None:
    92→    """
    93→    Normalize audio to 16kHz mono WAV, optionally trim silence, and write QC metrics.
    94→    """
    95→    out.mkdir(parents=True, exist_ok=True)
    96→
    97→    audio_files: list[Path] = []
    98→    for ext in ("*.wav", "*.mp3", "*.m4a", "*.flac", "*.ogg", "*.opus"):
    99→        audio_files.extend(sorted(segments_dir.rglob(ext)))
   100→    audio_files = sorted(set(audio_files))

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

from __future__ import annotations

import json
from pathlib import Path

import typer
from dotenv import load_dotenv

from maya_transcribe.audio import compute_qc, ffprobe, normalize_audio
from maya_transcribe.config import Settings
from maya_transcribe.dataset import find_segments_dir
from maya_transcribe.ids import segment_identity
from maya_transcribe.io import write_jsonl
from maya_transcribe.r2 import R2Client
from maya_transcribe.tar import safe_extract_tar

load_dotenv(dotenv_path=Path(".env"), override=False)

app = typer.Typer(add_completion=False, no_args_is_help=True)
bench_app = typer.Typer(add_completion=False, no_args_is_help=True, help="Benchmarking utilities.")
app.add_typer(bench_app, name="bench")

@app.command()
def download(
    video_id: str = typer.Option(..., help="YouTube video id (used to locate the tar in R2)."),
    out: Path = typer.Option(Path("data"), help="Output directory."),
    r2_key: str | None = typer.Option(None, help="Exact R2 object key (skips search)."),
    bucket: str | None = typer.Option(None, help="R2 bucket override (defaults to env R2_BUCKET)."),
    prefix: str | None = typer.Option(None, help="Optional R2 prefix override."),
    use_db: bool = typer.Option(False, help="Fetch `r2_tar_key` from Supabase Postgres (DATABASE_URL)."),
    dry_run: bool = typer.Option(False, help="Only show matched key(s); do not download."),
) -> None:
    """
    Download the cleaned/segmented tar for a video from R2 and extract it.
    """
    settings = Settings.load()
    r2 = R2Client.from_settings(settings)
    if bucket:
        r2 = R2Client(
            endpoint_url=r2.endpoint_url,
            access_key_id=r2.access_key_id,
            secret_access_key=r2.secret_access_key,
            bucket=bucket,
        )

    out.mkdir(parents=True, exist_ok=True)
    r2_prefix = prefix if prefix is not None else settings.r2_prefix

    matches: list[str]
    if use_db and not r2_key:
        from maya_transcribe.db import fetch_video_meta

        meta = fetch_video_meta(video_id)
        if meta and meta.r2_tar_key:
            r2_key = meta.r2_tar_key

    if r2_key:
        matches = [r2_key]
    else:
        best = r2.find_video_tar_key(video_id=video_id, prefix=r2_prefix)
        if not best:
            typer.echo("No matching objects found in R2.")
            raise typer.Exit(code=2)
        matches = [best]

    if dry_run:
        for key in matches:
            typer.echo(key)
        return

    key = matches[0]
    tar_path = r2.download_to(key=key, out_dir=out / video_id)
    extract_dir = out / video_id
    safe_extract_tar(tar_path, extract_dir=extract_dir)
    segments_dir = find_segments_dir(extract_dir)
    typer.echo(f"Downloaded: {tar_path}")
    typer.echo(f"Extracted to: {extract_dir}")
    if segments_dir:
        typer.echo(f"Segments dir: {segments_dir}")

@app.command()
def preprocess(
    segments_dir: Path = typer.Option(..., exists=True, file_okay=False, help="Directory containing audio segments."),
    out: Path = typer.Option(Path("outputs/preprocessed"), help="Output directory for normalized audio + QC."),
    sample_rate: int = typer.Option(16000, help="Target sample rate."),
    trim_silence: bool = typer.Option(True, help="Trim leading/trailing silence (ffmpeg silenceremove)."),
    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
) -> None:
    """
    Normalize audio to 16kHz mono WAV, optionally trim silence, and write QC metrics.
    """
    out.mkdir(parents=True, exist_ok=True)

    audio_files: list[Path] = []
    for ext in ("*.wav", "*.mp3", "*.m4a", "*.flac", "*.ogg", "*.opus"):
        audio_files.extend(sorted(segments_dir.rglob(ext)))
    audio_files = sorted(set(audio_files))


Let me check a few more utility files to complete the picture:


Bash

find /home/ubuntu/maya-transcribe/docs -type f 2>/dev/null | head -10

Check for documentation files


ls -la /home/ubuntu/maya-transcribe/maya_transcribe/ | grep -E "\.py$"

List all Python files in maya_transcribe package


     1→from __future__ import annotations
     2→
     3→import hashlib
     4→from dataclasses import dataclass
     5→from pathlib import Path
     6→
     7→
     8→@dataclass(frozen=True)
     9→class SegmentIdentity:
    10→    segment_id: str
    11→    base_id: str
    12→    relpath: str
    13→    uid: str
    14→
    15→
    16→def segment_identity(src: Path, root: Path) -> SegmentIdentity:
    17→    """
    18→    Produce a stable segment identity and prevent collisions when stems repeat.
    19→    - `base_id`: `src.stem`
    20→    - `relpath`: path relative to `root` (or absolute as fallback)
    21→    - `uid`: sha1(relpath) prefix
    22→    - `segment_id`: always includes uid for stability across runs
    23→    """
    24→    base_id = src.stem
    25→    try:
    26→        relpath = src.relative_to(root).as_posix()
    27→    except Exception:
    28→        relpath = str(src)
    29→
    30→    # Use 64 bits of hash to make collisions negligibly unlikely at large scale.
    31→    uid = hashlib.sha1(relpath.encode("utf-8")).hexdigest()[:16]
    32→    segment_id = f"{base_id}-{uid}"
    33→
    34→    return SegmentIdentity(segment_id=segment_id, base_id=base_id, relpath=relpath, uid=uid)
    35→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

from __future__ import annotations

import hashlib
from dataclasses import dataclass
from pathlib import Path

@dataclass(frozen=True)
class SegmentIdentity:
    segment_id: str
    base_id: str
    relpath: str
    uid: str

def segment_identity(src: Path, root: Path) -> SegmentIdentity:
    """
    Produce a stable segment identity and prevent collisions when stems repeat.
    - `base_id`: `src.stem`
    - `relpath`: path relative to `root` (or absolute as fallback)
    - `uid`: sha1(relpath) prefix
    - `segment_id`: always includes uid for stability across runs
    """
    base_id = src.stem
    try:
        relpath = src.relative_to(root).as_posix()
    except Exception:
        relpath = str(src)

    # Use 64 bits of hash to make collisions negligibly unlikely at large scale.
    uid = hashlib.sha1(relpath.encode("utf-8")).hexdigest()[:16]
    segment_id = f"{base_id}-{uid}"

    return SegmentIdentity(segment_id=segment_id, base_id=base_id, relpath=relpath, uid=uid)


     1→from __future__ import annotations
     2→
     3→import json
     4→from dataclasses import dataclass
     5→from pathlib import Path
     6→from typing import Any
     7→
     8→from jiwer import cer, wer
     9→
    10→from maya_transcribe.io import read_jsonl
    11→from maya_transcribe.textnorm import normalize_text
    12→
    13→
    14→@dataclass(frozen=True)
    15→class EvalRow:
    16→    segment_id: str
    17→    language: str | None
    18→    decision: str | None
    19→    duration_sec: float | None
    20→    duration_bucket: str | None
    21→    ref: str
    22→    hyp: str
    23→    wer: float
    24→    cer: float
    25→    oracle_best_wer: float | None = None
    26→    oracle_best_cer: float | None = None
    27→    oracle_best_key: str | None = None
    28→    selection_gap_wer: float | None = None
    29→
    30→
    31→def load_refs(refs_path: Path) -> dict[str, dict[str, Any]]:
    32→    """
    33→    Read a JSONL file of references.
    34→    Expected keys (any one): `text`, `transcript`, `reference`.
    35→    Required key: `segment_id` (or `id`).
    36→    """
    37→    refs: dict[str, dict[str, Any]] = {}
    38→    for row in read_jsonl(refs_path):
    39→        segment_id = str(row.get("segment_id") or row.get("id") or "").strip()
    40→        if not segment_id:
    41→            continue
    42→        refs[segment_id] = row
    43→    return refs
    44→
    45→
    46→def _get_text(row: dict[str, Any], keys: tuple[str, ...]) -> str | None:
    47→    for k in keys:
    48→        v = row.get(k)
    49→        if isinstance(v, str) and v.strip():
    50→            return v
    51→    return None
    52→
    53→
    54→def evaluate(
    55→    results_path: Path,
    56→    refs_path: Path,
    57→    out_path: Path | None = None,
    58→    hypothesis: str = "final",
    59→    oracle: bool = True,
    60→    language_source: str = "route",
    61→    decision_filter: list[str] | None = None,
    62→) -> dict[str, Any]:
    63→    refs = load_refs(refs_path)
    64→    results = read_jsonl(results_path)
    65→
    66→    decision_filter_set = None
    67→    if decision_filter:
    68→        decision_filter_set = {str(d).strip().lower() for d in decision_filter if str(d).strip()}
    69→
    70→    rows: list[EvalRow] = []
    71→    for r in results:
    72→        segment_id = str(r.get("segment_id") or "").strip()
    73→        if not segment_id or segment_id not in refs:
    74→            continue
    75→
    76→        ref_row = refs[segment_id]
    77→        ref_text = _get_text(ref_row, ("text", "transcript", "reference"))
    78→        if ref_text is None:
    79→            continue
    80→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

from __future__ import annotations

import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from jiwer import cer, wer

from maya_transcribe.io import read_jsonl
from maya_transcribe.textnorm import normalize_text

@dataclass(frozen=True)
class EvalRow:
    segment_id: str
    language: str | None
    decision: str | None
    duration_sec: float | None
    duration_bucket: str | None
    ref: str
    hyp: str
    wer: float
    cer: float
    oracle_best_wer: float | None = None
    oracle_best_cer: float | None = None
    oracle_best_key: str | None = None
    selection_gap_wer: float | None = None

def load_refs(refs_path: Path) -> dict[str, dict[str, Any]]:
    """
    Read a JSONL file of references.
    Expected keys (any one): `text`, `transcript`, `reference`.
    Required key: `segment_id` (or `id`).
    """
    refs: dict[str, dict[str, Any]] = {}
    for row in read_jsonl(refs_path):
        segment_id = str(row.get("segment_id") or row.get("id") or "").strip()
        if not segment_id:
            continue
        refs[segment_id] = row
    return refs

def _get_text(row: dict[str, Any], keys: tuple[str, ...]) -> str | None:
    for k in keys:
        v = row.get(k)
        if isinstance(v, str) and v.strip():
            return v
    return None

def evaluate(
    results_path: Path,
    refs_path: Path,
    out_path: Path | None = None,
    hypothesis: str = "final",
    oracle: bool = True,
    language_source: str = "route",
    decision_filter: list[str] | None = None,
) -> dict[str, Any]:
    refs = load_refs(refs_path)
    results = read_jsonl(results_path)

    decision_filter_set = None
    if decision_filter:
        decision_filter_set = {str(d).strip().lower() for d in decision_filter if str(d).strip()}

    rows: list[EvalRow] = []
    for r in results:
        segment_id = str(r.get("segment_id") or "").strip()
        if not segment_id or segment_id not in refs:
            continue

        ref_row = refs[segment_id]
        ref_text = _get_text(ref_row, ("text", "transcript", "reference"))
        if ref_text is None:
            continue


-rw-rw-r-- 1 ubuntu ubuntu  2181 Jan 13 04:24 align.py
-rw-rw-r-- 1 ubuntu ubuntu 11962 Jan 13 23:29 asr.py
-rw-rw-r-- 1 ubuntu ubuntu  4147 Jan 13 04:18 audio.py
-rw-rw-r-- 1 ubuntu ubuntu 14114 Jan 13 09:41 bench.py
-rw-rw-r-- 1 ubuntu ubuntu 20167 Jan 14 00:39 cli.py
-rw-rw-r-- 1 ubuntu ubuntu  1258 Jan 13 04:16 config.py
-rw-rw-r-- 1 ubuntu ubuntu 13909 Jan 13 07:09 ctc_fit.py
-rw-rw-r-- 1 ubuntu ubuntu   387 Jan 13 04:17 dataset.py
-rw-rw-r-- 1 ubuntu ubuntu  1508 Jan 13 04:37 db.py
-rw-rw-r-- 1 ubuntu ubuntu 11240 Jan 14 00:40 eval.py
-rw-rw-r-- 1 ubuntu ubuntu  9033 Jan 13 06:06 gemini_asr.py
-rw-rw-r-- 1 ubuntu ubuntu   992 Jan 13 06:16 ids.py
-rw-rw-r-- 1 ubuntu ubuntu    50 Jan 13 04:16 __init__.py
-rw-rw-r-- 1 ubuntu ubuntu   667 Jan 13 04:19 io.py
-rw-rw-r-- 1 ubuntu ubuntu  4782 Jan 13 23:24 language.py
-rw-rw-r-- 1 ubuntu ubuntu  2272 Jan 13 04:35 lid.py
-rw-rw-r-- 1 ubuntu ubuntu 22534 Jan 14 00:23 pipeline.py
-rw-rw-r-- 1 ubuntu ubuntu  3887 Jan 13 04:32 r2.py
-rw-rw-r-- 1 ubuntu ubuntu  6414 Jan 13 04:37 rover.py
-rw-rw-r-- 1 ubuntu ubuntu  1120 Jan 13 04:17 tar.py
-rw-rw-r-- 1 ubuntu ubuntu  1448 Jan 13 04:25 textnorm.py
-rw-rw-r-- 1 ubuntu ubuntu  6613 Jan 13 06:08 verify.py


/home/ubuntu/maya-transcribe/docs/Reference2.md
/home/ubuntu/maya-transcribe/docs/Reference6.md
/home/ubuntu/maya-transcribe/docs/Reference4.md
/home/ubuntu/maya-transcribe/docs/Reference5.md
/home/ubuntu/maya-transcribe/docs/Reference3.md
/home/ubuntu/maya-transcribe/docs/Reference1.md


Perfect! Let me create a comprehensive summary document. Let me read one more important file - the rest of cli.py for the full pipeline command:


   100→    audio_files = sorted(set(audio_files))
   101→
   102→    if not audio_files:
   103→        typer.echo(f"No audio files found under: {segments_dir}")
   104→        raise typer.Exit(code=2)
   105→
   106→    rows = []
   107→    norm_dir = out / "audio"
   108→    for src in audio_files:
   109→        ident = segment_identity(src, root=segments_dir)
   110→        segment_id = ident.segment_id
   111→        dst = norm_dir / f"{segment_id}.wav"
   112→        normalize_audio(
   113→            src,
   114→            dst,
   115→            sample_rate=sample_rate,
   116→            mono=True,
   117→            trim_silence=trim_silence,
   118→            silence_threshold_db=silence_threshold_db,
   119→            min_silence_sec=min_silence_sec,
   120→        )
   121→        info = ffprobe(dst)
   122→        qc = compute_qc(dst, sample_rate=sample_rate)
   123→        rows.append(
   124→            {
   125→                "segment_id": segment_id,
   126→                "base_id": ident.base_id,
   127→                "relpath": ident.relpath,
   128→                "uid": ident.uid,
   129→                "src_path": str(src),
   130→                "wav_path": str(dst),
   131→                "ffprobe": {
   132→                    "duration_sec": info.duration_sec,
   133→                    "sample_rate": info.sample_rate,
   134→                    "channels": info.channels,
   135→                },
   136→                "qc": {
   137→                    "duration_sec": qc.duration_sec,
   138→                    "rms_dbfs": qc.rms_dbfs,
   139→                    "peak_dbfs": qc.peak_dbfs,
   140→                    "clipping_fraction": qc.clipping_fraction,
   141→                },
   142→            }
   143→        )
   144→
   145→    write_jsonl(out / "qc.jsonl", rows)
   146→    typer.echo(f"Wrote: {out / 'qc.jsonl'}")
   147→    typer.echo(f"Normalized audio: {norm_dir}")
   148→
   149→
   150→@app.command()
   151→def lid(
   152→    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
   153→    model_id: str = typer.Option("facebook/mms-lid-256", help="HuggingFace model id."),
   154→    top_k: int = typer.Option(5, help="Return top-K predictions."),
   155→) -> None:
   156→    """
   157→    Spoken language identification (MMS-LID).
   158→    """
   159→    from maya_transcribe.lid import MMSLID
   160→
   161→    model = MMSLID(model_id=model_id)
   162→    res = model.predict(audio_path=audio, top_k=top_k)
   163→    for p in res.predictions:
   164→        typer.echo(f"{p.code}\t{p.prob:.4f}")
   165→
   166→
   167→@app.command()
   168→def asr(
   169→    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
   170→    language: str | None = typer.Option(None, help="Language hint (ISO-639-1 or ISO-639-3)."),
   171→    model: str = typer.Option("large-v3", help="Whisper model size or path for faster-whisper."),
   172→    beam_size: int = typer.Option(5, help="Beam size."),
   173→    best_of: int = typer.Option(5, help="Best-of (sampling candidates)."),
   174→    temperature: float = typer.Option(0.0, help="Temperature."),
   175→) -> None:
   176→    """
   177→    Transcribe a single file with faster-whisper.
   178→    """
   179→    from maya_transcribe.asr import FasterWhisperASR
   180→
   181→    asr_backend = FasterWhisperASR(model_size_or_path=model)
   182→    res = asr_backend.transcribe(
   183→        audio_path=audio, language=language, beam_size=beam_size, best_of=best_of, temperature=temperature
   184→    )
   185→    typer.echo(res.text)
   186→
   187→
   188→@app.command()
   189→def gemini(
   190→    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
   191→    language: str | None = typer.Option(None, help="Language hint (e.g., hin/tam/eng)."),
   192→    model: str = typer.Option("gemini-2.0-flash", help="Gemini model name."),
   193→    prompt_file: Path | None = typer.Option(Path("Prompt.md"), help="Prompt file (defaults to Prompt.md)."),
   194→    max_output_tokens: int = typer.Option(512, help="Max output tokens."),
   195→) -> None:
   196→    """
   197→    Transcribe a single file with Gemini (audio-native LLM). Output is parsed/validated strictly.
   198→    """
   199→    from maya_transcribe.gemini_asr import GeminiASR, GeminiConfig, load_prompt_template
   200→
   201→    prompt = load_prompt_template(prompt_file)
   202→    backend = GeminiASR(config=GeminiConfig(model=model, max_output_tokens=max_output_tokens), prompt_template=prompt)
   203→    res = backend.transcribe(audio_path=audio, language=language)
   204→    typer.echo(res.text)
   205→
   206→
   207→@app.command()
   208→def run(
   209→    segments_dir: Path = typer.Option(..., exists=True, file_okay=False, help="Directory containing audio segments."),
   210→    out: Path = typer.Option(Path("outputs/run"), help="Output directory."),
   211→    whisper_model: str = typer.Option("large-v3", help="Whisper model size/path for faster-whisper."),
   212→    lid_model_id: str = typer.Option("facebook/mms-lid-256", help="HuggingFace MMS-LID model id."),
   213→    require_lid_backend: bool = typer.Option(True, help="Fail if MMS-LID backend can't initialize."),
   214→    transformers_whisper_model: list[str] = typer.Option(
   215→        [], help="Extra whisper-family HF models to run via transformers (repeatable)."
   216→    ),
   217→    transformers_whisper_model_map: Path | None = typer.Option(
   218→        None, help="JSON map {lang_code: hf_model_id} for per-language Whisper-family HF models (transformers)."
   219→    ),
   220→    gemini_model: str | None = typer.Option(None, help="Optional Gemini model name (adds Gemini as a hypothesis)."),
   221→    gemini_prompt_file: Path | None = typer.Option(Path("Prompt.md"), help="Prompt file for Gemini."),
   222→    gemini_hard_only: bool = typer.Option(True, help="Only call Gemini on 'hard' segments."),
   223→    ctc_model: str | None = typer.Option(None, help="Optional HF CTC model id for audio-grounded fit scoring."),
   224→    ctc_model_map: Path | None = typer.Option(None, help="JSON map {lang_code: hf_model_id} for CTC scoring."),
   225→    require_ctc: bool = typer.Option(True, help="When CTC is enabled, require a CTC score to accept."),
   226→    min_ctc_margin: float = typer.Option(0.05, help="Min (second-best - best) CTC loss margin to auto-accept."),
   227→    trim_silence: bool = typer.Option(True, help="Trim leading/trailing silence (ffmpeg silenceremove)."),
   228→    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
   229→    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
   230→    limit: int | None = typer.Option(None, help="Process only the first N segments."),
   231→    thresholds_json: Path | None = typer.Option(None, help="Optional JSON file of threshold overrides (verify + routing)."),
   232→) -> None:
   233→    """
   234→    End-to-end pipeline: preprocess -> LID -> ASR ensemble -> ROVER -> verify.
   235→    """
   236→    from maya_transcribe.pipeline import run_pipeline
   237→
   238→    ctc_map = None
   239→    if ctc_model_map is not None:
   240→        ctc_map = json.loads(ctc_model_map.read_text(encoding="utf-8"))
   241→        if not isinstance(ctc_map, dict):
   242→            raise typer.BadParameter("ctc_model_map must be a JSON object {lang_code: model_id}")
   243→
   244→    tf_map = None
   245→    if transformers_whisper_model_map is not None:
   246→        tf_map = json.loads(transformers_whisper_model_map.read_text(encoding="utf-8"))
   247→        if not isinstance(tf_map, dict):
   248→            raise typer.BadParameter("transformers_whisper_model_map must be a JSON object {lang_code: model_id}")
   249→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

    audio_files = sorted(set(audio_files))

    if not audio_files:
        typer.echo(f"No audio files found under: {segments_dir}")
        raise typer.Exit(code=2)

    rows = []
    norm_dir = out / "audio"
    for src in audio_files:
        ident = segment_identity(src, root=segments_dir)
        segment_id = ident.segment_id
        dst = norm_dir / f"{segment_id}.wav"
        normalize_audio(
            src,
            dst,
            sample_rate=sample_rate,
            mono=True,
            trim_silence=trim_silence,
            silence_threshold_db=silence_threshold_db,
            min_silence_sec=min_silence_sec,
        )
        info = ffprobe(dst)
        qc = compute_qc(dst, sample_rate=sample_rate)
        rows.append(
            {
                "segment_id": segment_id,
                "base_id": ident.base_id,
                "relpath": ident.relpath,
                "uid": ident.uid,
                "src_path": str(src),
                "wav_path": str(dst),
                "ffprobe": {
                    "duration_sec": info.duration_sec,
                    "sample_rate": info.sample_rate,
                    "channels": info.channels,
                },
                "qc": {
                    "duration_sec": qc.duration_sec,
                    "rms_dbfs": qc.rms_dbfs,
                    "peak_dbfs": qc.peak_dbfs,
                    "clipping_fraction": qc.clipping_fraction,
                },
            }
        )

    write_jsonl(out / "qc.jsonl", rows)
    typer.echo(f"Wrote: {out / 'qc.jsonl'}")
    typer.echo(f"Normalized audio: {norm_dir}")

@app.command()
def lid(
    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
    model_id: str = typer.Option("facebook/mms-lid-256", help="HuggingFace model id."),
    top_k: int = typer.Option(5, help="Return top-K predictions."),
) -> None:
    """
    Spoken language identification (MMS-LID).
    """
    from maya_transcribe.lid import MMSLID

    model = MMSLID(model_id=model_id)
    res = model.predict(audio_path=audio, top_k=top_k)
    for p in res.predictions:
        typer.echo(f"{p.code}\t{p.prob:.4f}")

@app.command()
def asr(
    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
    language: str | None = typer.Option(None, help="Language hint (ISO-639-1 or ISO-639-3)."),
    model: str = typer.Option("large-v3", help="Whisper model size or path for faster-whisper."),
    beam_size: int = typer.Option(5, help="Beam size."),
    best_of: int = typer.Option(5, help="Best-of (sampling candidates)."),
    temperature: float = typer.Option(0.0, help="Temperature."),
) -> None:
    """
    Transcribe a single file with faster-whisper.
    """
    from maya_transcribe.asr import FasterWhisperASR

    asr_backend = FasterWhisperASR(model_size_or_path=model)
    res = asr_backend.transcribe(
        audio_path=audio, language=language, beam_size=beam_size, best_of=best_of, temperature=temperature
    )
    typer.echo(res.text)

@app.command()
def gemini(
    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
    language: str | None = typer.Option(None, help="Language hint (e.g., hin/tam/eng)."),
    model: str = typer.Option("gemini-2.0-flash", help="Gemini model name."),
    prompt_file: Path | None = typer.Option(Path("Prompt.md"), help="Prompt file (defaults to Prompt.md)."),
    max_output_tokens: int = typer.Option(512, help="Max output tokens."),
) -> None:
    """
    Transcribe a single file with Gemini (audio-native LLM). Output is parsed/validated strictly.
    """
    from maya_transcribe.gemini_asr import GeminiASR, GeminiConfig, load_prompt_template

    prompt = load_prompt_template(prompt_file)
    backend = GeminiASR(config=GeminiConfig(model=model, max_output_tokens=max_output_tokens), prompt_template=prompt)
    res = backend.transcribe(audio_path=audio, language=language)
    typer.echo(res.text)

@app.command()
def run(
    segments_dir: Path = typer.Option(..., exists=True, file_okay=False, help="Directory containing audio segments."),
    out: Path = typer.Option(Path("outputs/run"), help="Output directory."),
    whisper_model: str = typer.Option("large-v3", help="Whisper model size/path for faster-whisper."),
    lid_model_id: str = typer.Option("facebook/mms-lid-256", help="HuggingFace MMS-LID model id."),
    require_lid_backend: bool = typer.Option(True, help="Fail if MMS-LID backend can't initialize."),
    transformers_whisper_model: list[str] = typer.Option(
        [], help="Extra whisper-family HF models to run via transformers (repeatable)."
    ),
    transformers_whisper_model_map: Path | None = typer.Option(
        None, help="JSON map {lang_code: hf_model_id} for per-language Whisper-family HF models (transformers)."
    ),
    gemini_model: str | None = typer.Option(None, help="Optional Gemini model name (adds Gemini as a hypothesis)."),
    gemini_prompt_file: Path | None = typer.Option(Path("Prompt.md"), help="Prompt file for Gemini."),
    gemini_hard_only: bool = typer.Option(True, help="Only call Gemini on 'hard' segments."),
    ctc_model: str | None = typer.Option(None, help="Optional HF CTC model id for audio-grounded fit scoring."),
    ctc_model_map: Path | None = typer.Option(None, help="JSON map {lang_code: hf_model_id} for CTC scoring."),
    require_ctc: bool = typer.Option(True, help="When CTC is enabled, require a CTC score to accept."),
    min_ctc_margin: float = typer.Option(0.05, help="Min (second-best - best) CTC loss margin to auto-accept."),
    trim_silence: bool = typer.Option(True, help="Trim leading/trailing silence (ffmpeg silenceremove)."),
    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
    limit: int | None = typer.Option(None, help="Process only the first N segments."),
    thresholds_json: Path | None = typer.Option(None, help="Optional JSON file of threshold overrides (verify + routing)."),
) -> None:
    """
    End-to-end pipeline: preprocess -> LID -> ASR ensemble -> ROVER -> verify.
    """
    from maya_transcribe.pipeline import run_pipeline

    ctc_map = None
    if ctc_model_map is not None:
        ctc_map = json.loads(ctc_model_map.read_text(encoding="utf-8"))
        if not isinstance(ctc_map, dict):
            raise typer.BadParameter("ctc_model_map must be a JSON object {lang_code: model_id}")

    tf_map = None
    if transformers_whisper_model_map is not None:
        tf_map = json.loads(transformers_whisper_model_map.read_text(encoding="utf-8"))
        if not isinstance(tf_map, dict):
            raise typer.BadParameter("transformers_whisper_model_map must be a JSON object {lang_code: model_id}")


Perfect! Now I have a comprehensive understanding of the codebase. Let me provide a thorough summary:

## Comprehensive Maya-Transcribe Codebase Analysis

### 1. Overall Directory Structure

```
/home/ubuntu/maya-transcribe/
├── maya_transcribe/              # Main package directory
│   ├── __init__.py
│   ├── pipeline.py               # Main end-to-end pipeline orchestration
│   ├── cli.py                    # Command-line interface (typer-based)
│   ├── config.py                 # R2 S3-compatible credentials config
│   ├── asr.py                    # ASR backends (FasterWhisper, TransformersWhisper)
│   ├── lid.py                    # Language ID (MMS-LID)
│   ├── rover.py                  # ROVER system combination
│   ├── verify.py                 # Audio-grounded verification gate
│   ├── ctc_fit.py                # CTC fit scoring (audio verification)
│   ├── textnorm.py               # Text normalization (NFKC, digit conversion)
│   ├── language.py               # Language code mappings, script detection
│   ├── audio.py                  # Audio processing (ffmpeg/ffprobe integration)
│   ├── align.py                  # Token-level Levenshtein alignment
│   ├── gemini_asr.py             # Gemini audio-native LLM integration
│   ├── ids.py                    # Segment identity & collision prevention
│   ├── eval.py                   # Evaluation (WER/CER computation)
│   ├── r2.py                     # R2/S3-compatible cloud storage client
│   ├── db.py                     # Database integration (PostgreSQL)
│   ├── dataset.py                # Dataset utilities
│   ├── bench.py                  # Benchmarking suite
│   ├── io.py                     # JSONL I/O utilities
│   └── tar.py                    # Tar extraction utilities
├── data/                         # Test data & manifests
├── outputs/                      # Pipeline outputs (results.jsonl, summary.json)
├── docs/                         # Reference documentation
├── pyproject.toml                # Package metadata & dependencies
├── README.md                     # User guide
├── master.md                     # Executive technical summary & architecture
├── Agents.md                     # Detailed decision rationale
├── Prompt.md                     # Gemini prompt template
├── .env                          # R2 credentials
└── task.md                       # Original requirements
```

---

### 2. Python Files & Their Purposes

| File | Purpose | Key Classes/Functions |
|------|---------|----------------------|
| **pipeline.py** (496 lines) | Main orchestration engine; calls all subsystems in sequence | `run_pipeline()`, `_iter_audio_files()`, `_asr_summary()` |
| **asr.py** (349 lines) | ASR backends for diverse hypothesis generation | `FasterWhisperASR`, `TransformersWhisperASR`, `TransformersWhisperASRRouter`, `ASRResult`, `ASRSegment` |
| **lid.py** (69 lines) | Language identification via MMS-LID (facebook/mms-lid-256) | `MMSLID`, `LIDResult`, `LanguagePrediction` |
| **rover.py** (199 lines) | System combination using ROVER-style voting on aligned tokens | `rover_combine()`, `choose_centroid()`, `RoverResult`, `RoverDisagreement` |
| **verify.py** (160 lines) | Precision-first verification gate (conservative acceptance) | `verify_segment()`, `lid_confidence()`, `Verification` |
| **ctc_fit.py** (393 lines) | Audio-grounded fit scoring via CTC loss & forced alignment | `CTCFitScorer`, `_CTCModel`, `rank_candidates_by_ctc()`, `CTCScore` |
| **textnorm.py** (65 lines) | Conservative normalization: NFKC, drop punctuation, normalize digits | `normalize_text()`, `tokenize_words()` |
| **language.py** (188 lines) | Language code mappings, script detection, Whisper lang conversion | `expected_script()`, `dominant_script()`, `to_whisper_lang()`, `LANG_CODE_TO_SCRIPT` |
| **audio.py** (140 lines) | Audio processing: ffmpeg/ffprobe integration, QC metrics | `decode_pcm16()`, `normalize_audio()`, `compute_qc()`, `ffprobe()`, `AudioQCMetrics` |
| **align.py** (77 lines) | Token-level Levenshtein alignment for ROVER | `align_tokens()`, `edit_distance()`, `AlignmentResult` |
| **gemini_asr.py** (160+ lines) | Gemini 2.0-flash audio-native LLM transcription | `GeminiASR`, `GeminiConfig`, `load_prompt_template()`, `parse_strict_output()` |
| **eval.py** (147+ lines) | WER/CER evaluation against human references (jiwer-based) | `evaluate()`, `load_refs()`, `EvalRow` |
| **cli.py** (320+ lines) | Typer CLI with commands: download, preprocess, lid, asr, gemini, run, eval, bench | `@app.command` decorated functions |
| **config.py** (44 lines) | R2 S3-compatible credentials & settings | `Settings`, `.env` loading |
| **ids.py** (35 lines) | Stable segment identity generation (SHA1 collision prevention) | `segment_identity()`, `SegmentIdentity` |
| **ctc_fit.py** (detailed above) | --- | --- |
| **r2.py** | R2 cloud storage client (boto3-based) | `R2Client` |
| **db.py** | Supabase PostgreSQL integration (optional) | `fetch_video_meta()` |
| **dataset.py** | Segment directory discovery | `find_segments_dir()` |
| **bench.py** | Benchmarking suite (HuggingFace datasets, model comparisons) | Benchmark orchestration |
| **io.py** | JSONL read/write utilities | `read_jsonl()`, `write_jsonl()` |
| **tar.py** | Tar extraction with security checks | `safe_extract_tar()` |

---

### 3. Configuration Files

| File | Content | Purpose |
|------|---------|---------|
| **pyproject.toml** | Python package metadata, optional dependencies | Define package: `maya-transcribe`, dependencies grouped: `[asr]`, `[torch]`, `[db]`, `[gemini]`, `[bench]`, `[full]` |
| **.env** | R2 credentials (R2_ENDPOINT_URL, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, R2_BUCKET, R2_PREFIX) | Cloud storage auth (Cloudflare R2 or S3-compatible) |
| **Prompt.md** | Gemini system prompt for strict transcription | Audio-native LLM instruction template |
| **master.md** | Architecture document & decision rationale | High-level strategy, pipeline block diagram, model stack |
| **Agents.md** | Detailed analysis of design choices | Per-component justification |
| **task.md** | Original project requirements | Specification |
| **README.md** | User setup & usage guide | CLI examples, installation instructions |

---

### 4. Main Pipeline Implementation (`pipeline.py`)

The `run_pipeline()` function is the orchestration core. Here's the execution sequence:

#### **Step 1: Audio Preprocessing**
- Input: Raw audio file (wav, mp3, m4a, flac, ogg, opus)
- Operations:
  - `ffprobe()`: Extract metadata (duration, sample rate, channels)
  - `normalize_audio()`: Convert to 16 kHz mono WAV via ffmpeg silenceremove filter
  - `compute_qc()`: Compute audio QC metrics (duration, RMS dBFS, peak dBFS, clipping fraction)
- Output: Normalized WAV + QC dict

#### **Step 2: Spoken Language Identification (LID)**
- Model: **MMS-LID** (facebook/mms-lid-256) — multilingual spoken language classifier
- Input: Normalized 16 kHz mono WAV
- Output: `LIDResult` with top-5 predictions + probabilities
- Routing Logic:
  - If `top1_prob >= 0.90` AND `(top1_prob - top2_prob) >= 0.20` → confident route
  - Else fallback to path-based inference or uncertain route (multi-lang ASR)
- Stored in: `route.language_hint`, `route.language_hint_source` (lid|path|none)

#### **Step 3: ASR Ensemble (Diverse Hypothesis Generation)**
Multiple backends run in parallel on the same audio:

1. **FasterWhisper** (facebook/whisper-large-v3)
   - Deterministic run: `beam_size=5, best_of=5, temperature=0.0`
   - Sampling run: `beam_size=5, best_of=10, temperature=0.2` (if `whisper_run_sampling=True`)

2. **TransformersWhisper** (HuggingFace Whisper-family checkpoints)
   - Per-language fine-tuned models can be added via CLI: `--transformers-whisper-model <hf_model_id>`
   - Router option: `--transformers-whisper-model-map {lang: model_id}` for dynamic model selection

3. **Gemini 2.0-flash** (optional, audio-native LLM)
   - Runs only on "hard" segments (low logprob, high compression ratio, or uncertain language)
   - Gating: `gemini_hard_only=True` (default)

4. **Error Handling**: If any backend fails, it's logged to `asr_errors` dict; pipeline continues if ≥1 backend succeeds

**Result Storage**: Dict of `{model_key: ASRResult}` where each `ASRResult` contains:
- `text`: Transcribed string
- `model`: Backend identifier (e.g., "faster-whisper:large-v3", "transformers:facebook/wav2vec2-indic")
- `language`: Language code (if detectable)
- `language_probability`: Confidence (if available)
- `segments`: List of segment-level timing info
- `meta`: Decode parameters

#### **Step 4: Text Normalization**
- Function: `normalize_text(text, lang=language_hint)`
- Process:
  - Unicode NFKC normalization
  - Remove punctuation (Unicode category P)
  - Normalize all Unicode digits to ASCII (0–9)
  - Collapse whitespace
  - Lowercase only for English (`lang='en'` or `'eng'`)
- Purpose: Enable fair comparison across models without phonetic/orthographic differences

#### **Step 5: ROVER System Combination**
- Input: Normalized hypotheses dict `{model_key: text}`
- Algorithm (lightweight ROVER variant):
  1. **Choose centroid**: Find hypothesis with minimal average Levenshtein distance to others
  2. **Align others**: Align all hypotheses to skeleton using token-level Levenshtein alignment
  3. **Vote per position**: For each skeleton token, take vote (weighted or uniform)
  4. **Gap insertions**: For insertions between skeleton tokens, keep centroid insertion if weight ≥ 50%
  5. **Disagreement tracking**: Log all positions where models disagree (for audit trails)
- Output: `RoverResult` with:
  - `text`: Combined hypothesis
  - `skeleton_key`: Which model was used as skeleton
  - `disagreement_rate`: Fraction of positions with disagreement
  - `disagreements`: List of `RoverDisagreement` objects (kind=token|gap, index, chosen, options)
- Optional Filtering:
  - `filter_hypotheses_by_script`: Drop hypotheses in wrong script (e.g., Devanagari output for Malayalam)
  - `rover_prefer_router`: Prioritize language-routed model's hypothesis
  - `rover_include_sampling`: Include or exclude sampling diversity pass

#### **Step 6: Audio-Grounded CTC Verification** (Optional)
- Function: `rank_candidates_by_ctc(scorer, audio_path, candidates, lang)`
- Input: Audio + text candidates (`{key: text}`)
- Process (per candidate):
  1. Load CTC model (language-specific or default)
  2. Compute log-probs from audio via CTC encoder
  3. Tokenize text (with lang-specific target language setting)
  4. Compute CTC loss (reduced to mean over target length)
  5. Attempt forced alignment to extract per-token scores + coverage
  6. Detect span mismatches (alignment spans ≠ tokenized targets)
- Output: Ranked scores with:
  - `best_key`: Candidate with lowest CTC loss
  - `margin`: (second_best_loss - best_loss)
  - `scores[key]`: {"loss", "token_coverage", "token_score_mean", "span_mismatch", ...}
- Decision: If `margin >= min_ctc_margin`, use best candidate; else set decision to "human"

#### **Step 7: Verification Gate** (Precision-First)
- Function: `verify_segment(..., qc, lid, final_text, rover, asr_summaries, ctc, thresholds)`
- Decision logic (conservative):
  - **REJECT** if: too short (<0.8s), no speech detected (>0.8 prob), extreme clipping
  - **HUMAN** (flag for manual review) if:
    - LID uncertain (low probability or margin)
    - ASR disagreement high (ROVER disagreement_rate > 0.03)
    - Low ASR confidence (avg logprob < -1.0 or compression_ratio > 2.4)
    - CTC fit poor (margin < 0.05, token_coverage < 0.85, span_mismatch=True)
    - Contains `<unk>` or `<inaudible>` tokens
  - **ACCEPT** if: passes all checks
- Output: `Verification` with `decision`, `reasons` (list of failure codes), `scores` (for audit)

#### **Step 8: Output Schema**
Results are written to `/outputs/{job_id}/results.jsonl` as one JSON per segment:

```json
{
  "schema_version": 1,
  "segment_id": "audio_name-<16-char-hash>",
  "base_id": "audio_name",
  "relpath": "relative/path/in/segments/dir",
  "uid": "<16-char-hash>",
  "src_path": "/path/to/original/audio",
  "wav_path": "/path/to/normalized/audio.wav",
  "src_ffprobe": {...},
  "ffprobe": {...},
  "qc": {"duration_sec": 5.2, "rms_dbfs": -20.5, "peak_dbfs": -5.1, "clipping_fraction": 0.0},
  "lid": {"model": "facebook/mms-lid-256", "predictions": [{"code": "mal", "prob": 0.95}, ...]},
  "route": {"language_hint": "mal", "language_hint_source": "lid"},
  "asr": {
    "faster-whisper:large-v3|det": {
      "model": "faster-whisper:large-v3|det",
      "text": "...",
      "language": "ml",
      "language_probability": 0.97,
      "segments": [...segment-level timing...],
      "meta": {"device": "cuda", "beam_size": 5, "temperature": 0.0, ...}
    },
    "transformers:facebook/indic_whisper_v1|mal": {...},
    ...
  },
  "asr_summary": {
    "faster-whisper:large-v3|det": {
      "language": "ml",
      "language_probability": 0.97,
      "avg_logprob_mean": -0.35,
      "no_speech_prob_mean": 0.01,
      "compression_ratio_mean": 1.2
    },
    ...
  },
  "asr_errors": {"some_backend": "RuntimeError: CUDA out of memory"},
  "ensemble": {
    "unique_models": ["faster-whisper:large-v3", "transformers:facebook/indic_whisper_v1"],
    "unique_backends": ["faster-whisper", "transformers"],
    "warnings": ["low_model_diversity"]
  },
  "normalized": {
    "faster-whisper:large-v3|det": "normalized text",
    "transformers:...": "normalized text"
  },
  "rover": {
    "text": "ROVER combined text",
    "skeleton_key": "faster-whisper:large-v3|det",
    "disagreement_rate": 0.02,
    "inputs": ["faster-whisper:large-v3|det", "transformers:..."],
    "filtered_out": [],
    "disagreements": [
      {"kind": "token", "index": 5, "chosen": "word_a", "options": {"model_x": "word_a", "model_y": "word_b"}},
      ...
    ]
  },
  "ctc": {
    "model_id": "facebook/wav2vec2-indic-1billion",
    "best_key": "faster-whisper:large-v3|det",
    "best_loss": 0.32,
    "second_best_loss": 0.38,
    "margin": 0.06,
    "scores": {
      "faster-whisper:large-v3|det": {
        "loss": 0.32,
        "token_coverage": 0.92,
        "token_score_mean": -0.8,
        "span_mismatch": false
      },
      ...
    }
  },
  "final": {
    "text": "final accepted or human-reviewed text",
    "source": "rover|asr:model_key|ctc"
  },
  "verification": {
    "decision": "accept|human|reject",
    "reasons": ["lid_uncertain", "asr_disagreement"],
    "scores": {
      "duration_sec": 5.2,
      "clipping_fraction": 0.0,
      "rover_disagreement_rate": 0.02,
      "lid": {"available": true, "top1": "mal", "top1_prob": 0.95, "margin": 0.18, "confident": true},
      "no_speech_prob_max": 0.02,
      "avg_logprob_mean": -0.35,
      "compression_ratio_mean": 1.2,
      "ctc": {...}
    }
  },
  "config": {...},
  "timing": {
    "preprocess_sec": 0.5,
    "lid_sec": 0.3,
    "asr_sec": 2.1,
    "normalize_sec": 0.01,
    "rover_sec": 0.02,
    "ctc_sec": 0.4,
    "verify_sec": 0.01,
    "total_sec": 3.3
  }
}
```

Also written: `/outputs/{job_id}/summary.json`:
```json
{
  "segments_total": 100,
  "counts": {"accept": 87, "human": 10, "reject": 3, "error": 0},
  "avg_total_sec": 3.4,
  "p50_total_sec": 3.2,
  "p90_total_sec": 4.1
}
```

---

### 5. Key Components Deep Dive

#### **A. Language Identification (LID) - `lid.py`**

```python
class MMSLID:
    def __init__(self, model_id="facebook/mms-lid-256", device=None):
        # Load MMS-LID: 256-language audio classifier from HuggingFace
        # Auto-selects CUDA if available

    def predict(self, audio_path: Path, top_k=5) -> LIDResult:
        # Input: 16 kHz mono WAV
        # Output: Top-K (lang_code, probability) predictions
        # Returns: LIDResult(model, [LanguagePrediction(code, prob), ...])
```

**Key Characteristics:**
- Supports 256 languages via single model
- Returns calibrated probabilities
- Margin-based confidence thresholding in pipeline

#### **B. ASR Backends - `asr.py`**

**FasterWhisperASR:**
- Uses `faster-whisper` (ONNX-optimized Whisper inference)
- Model sizes: tiny, base, small, medium, large, large-v3
- Parameters: `beam_size`, `best_of`, `temperature`, `vad_filter`
- Lightweight, fast inference on CPU/GPU
- Language support: Whisper's multilingual vocab (50+ languages)

**TransformersWhisperASR:**
- Uses HuggingFace `transformers` library
- Enables fine-tuned checkpoints (e.g., IndicWhisper, per-language models)
- Supports forced language decoding via `forced_decoder_ids`
- Handles HF model versioning & dtype management

**TransformersWhisperASRRouter:**
- Lazy-loads language-specific models on demand
- LRU cache: keeps only `max_loaded_models` in memory (default 1)
- Falls back to default model if language not in map
- Prevents OOM on multi-model runs

**Key Outputs (ASRResult):**
- `text`: Joined transcription
- `language`: Detected language (Whisper's own detection or forced)
- `segments`: List of (start, end, text, avg_logprob, no_speech_prob, compression_ratio)
- `meta`: Decode params for audit

#### **C. ROVER System Combination - `rover.py`**

**Algorithm:**
1. **Centroid Selection** (`choose_centroid(hyps)`):
   - For each hypothesis, compute avg normalized Levenshtein distance to all others
   - Skeleton = hypothesis with minimal average distance (least divergent)

2. **Alignment** (`align_tokens(ref, hyp)`):
   - Standard Levenshtein alignment with traceback
   - Returns: edit distance + alignment as `[(ref_tok|None, hyp_tok|None), ...]`

3. **Voting**:
   - For each skeleton position, collect votes from aligned hypotheses
   - Weighted voting (if weights provided; default uniform)
   - Tie-breaking: prefer skeleton token if in top votes, else alphabetical

4. **Gap Insertions**:
   - Between skeleton tokens, insertions are candidates
   - Centroid gap kept if weight ≥ 50% of total
   - Prevents spurious insertions from outlier models

5. **Disagreement Tracking**:
   - Logs every position where models differ
   - Stores: kind (token|gap), index, chosen, options dict

**Example:**
```
Hypotheses:
  model_a: "நல்ல நாள்"
  model_b: "நல்ல நாள்"
  model_c: "நல்ல நாள் உண்ட"

Skeleton: model_a (closest to others)
Alignment:
  model_a → (நல்ல, நல்ல), (நாள், நாள்)
  model_b → (நல்ல, நல்ல), (நாள், நாள்)
  model_c → (நல்ல, நல்ல), (நாள், நாள்), gap_insert: உண்ட (1/3 weight, <50% → dropped)

Result: "நல்ல நாள்"
Disagreement rate: 0.0 (if no ties; >0 if model_c voted for உண்ட)
```

#### **D. CTC Audio-Grounded Verification - `ctc_fit.py`**

**Process:**
1. Load CTC model (language-aware, e.g., `facebook/wav2vec2-indic-1billion`)
2. Encode audio → log-probabilities (B×T×C)
3. Tokenize text (with language-specific vocab if supported)
4. Compute CTC loss: `F.ctc_loss(log_probs, target_ids, reduction='mean')`
   - Mean reduction: loss normalized by target length (enables cross-length comparison)
5. **Forced Alignment** (`torchaudio.functional.forced_align`):
   - Finds token-level alignments with confidence scores
   - Merges adjacent tokens → spans
   - Computes `token_coverage`: fraction of tokens with score ≥ -3.0
   - Detects `span_mismatch`: number of spans ≠ number of tokens (alignment failure signal)

**Key Heuristics:**
- `unk_ratio`: % of unknown tokens in tokenization (max 20% allowed)
- `token_coverage`: % of tokens with good alignment score (must be ≥ 85% for accept)
- `token_score_mean`: average alignment score (must be ≥ -3.0)
- `span_mismatch`: True if alignment spans don't match token count (phoneme-text mismatch)

**Why CTC fit matters:**
- Prevents "fluent hallucinations": text can sound plausible but not match audio
- Language-agnostic: any language with a CTC model can be verified
- Independent from ASR: catches errors that all ASR models agree on

#### **E. Text Normalization - `textnorm.py`**

**Conservative approach** (does NOT alter meaning, only align differences):
1. **Unicode normalization**: NFKC (compatibility form)
2. **Remove punctuation**: Drop Unicode category P (all marks)
3. **Normalize digits**: Convert any Unicode digit (e.g., ०, ਪ) → ASCII 0-9
4. **Collapse whitespace**: Multiple spaces → single space
5. **Language-specific lowercasing**: English only (preserve case for scripts like Devanagari)

**Purpose**: Enable ROVER voting without artificial differences from punctuation/digit encoding

#### **F. Language Code & Script Mapping - `language.py`**

**Supported Languages (12 Indic + English):**
- ISO-639-3 ↔ ISO-639-1 ↔ Whisper code mappings
- Examples: `mal` ↔ `ml` (Malayalam), `hin` ↔ `hi` (Hindi)

**Script Detection:**
- `dominant_script(text)`: Heuristic scan of Unicode blocks (Devanagari, Bengali, etc.)
- `expected_script(lang_code)`: Returns canonical script for language
- Used to filter "wrong script" ASR outputs (e.g., Devanagari decode for Malayalam)

#### **G. Verification Gate - `verify.py`**

**Thresholds (all configurable):**
- `min_duration_sec`: 0.8s (reject too short)
- `max_clipping_fraction`: 0.01 (flag clipping as "human")
- `lid_min_prob`: 0.90 (LID confidence threshold)
- `lid_min_margin`: 0.20 (LID margin between top-2)
- `max_rover_disagreement_rate`: 0.03 (flag high disagreement)
- `min_avg_logprob_mean`: -1.0 (Whisper hallucination heuristic)
- `max_compression_ratio_mean`: 2.4 (Whisper repetition heuristic)
- CTC thresholds: `require_ctc`, `min_ctc_margin`, `min_ctc_token_coverage`

**Decision Logic:**
- Accept: Passes all thresholds
- Human: Flags for manual review (soft failures, ambiguity)
- Reject: Hard failures (too short, no speech, extreme clipping)

---

### 6. CLI Interface (`cli.py`)

**Main Commands:**

| Command | Purpose |
|---------|---------|
| `download` | Fetch tar from R2, extract segments |
| `preprocess` | Audio normalization + QC metrics |
| `lid` | Single-file language ID |
| `asr` | Single-file transcription (faster-whisper) |
| `gemini` | Single-file Gemini LLM transcription |
| **`run`** | **End-to-end pipeline** |
| `eval` | Compute WER/CER against gold refs |
| `bench` | Benchmark multiple models on test suite |

**`run` Command Parameters:**
- Audio input: `--segments-dir`
- Output: `--out`
- LID: `--lid-model-id`
- ASR: `--whisper-model`, `--transformers-whisper-model` (repeatable), `--transformers-whisper-model-map`
- Optional LLM: `--gemini-model`, `--gemini-hard-only`
- Audio verification: `--ctc-model`, `--ctc-model-map`, `--require-ctc`, `--min-ctc-margin`
- Audio QC: `--trim-silence`, `--silence-threshold-db`, `--min-silence-sec`
- Thresholds override: `--thresholds-json` (JSON file)
- Debugging: `--limit` (process only first N)

---

### 7. Model Stack (From `master.md`)

**Recommended Ensemble:**

| Component | Models | Purpose |
|-----------|--------|---------|
| **LID** | MMS-LID (facebook/mms-lid-256) | 256-language spoken language ID |
| **ASR (Indic-confident)** | IndicWhisper + Whisper large-v3 + IndicConformer-600M | Specialist fine-tuned + robust generalist + diverse architecture |
| **ASR (uncertain)** | Whisper large-v3 + IndicConformer-600M + MMS ASR | Multilingual generalists |
| **ASR (English)** | Whisper large-v3 + Voxtral (optional) | English generalist |
| **System Combination** | ROVER (weighted voting on tokens) | Exploit diversity, reduce WER |
| **Audio Verification** | CTC fit (wav2vec2 multilingual) | Detect hallucinations, enforce audio-text alignment |

**Optional Add-ons:**
- **Gemini 2.0-flash**: Audio-native LLM for "hard" segments (uncertain language, low confidence, high repetition)
- **Per-language routers**: Map language → best fine-tuned model dynamically

---

### 8. Key Design Decisions

| Aspect | Choice | Rationale |
|--------|--------|-----------|
| **Offline first** | No streaming; full audio buffered | Enables ensemble, ROVER, CTC verification |
| **Precision > Recall** | Conservative verification gate | Minimize label noise in training data |
| **Diversity over winners** | Ensemble + ROVER vs. "pick best model" | Errors differ across architectures; voting recovers missed content |
| **Token-level ROVER** | Lightweight alignment + voting | Faster than full confusion-network; still recovers fine-grained diversity |
| **CTC as verification** | Independent from ASR | Catches hallucinations all models agree on (fluent nonsense) |
| **Language routing** | Confident LID → single-language ASR | Prevents script/vocab mismatch; faster inference |
| **Conservative normalization** | NFKC + drop punctuation only | Preserve script/phonology; don't hallucinate corrections |
| **Forced alignment** | Torchaudio `forced_align` | Detect when text doesn't span audio (span_mismatch) |
| **Human-in-loop tail** | Only flag uncertain/ambiguous | Reviewers correct mistakes humans can catch in a few seconds |

---

### 9. Output & Evaluation

**Results Format:** JSONL with full audit trail (all hypotheses, scores, reasoning)

**Evaluation (`eval.py`):**
- Compares `final.text` against human references
- Computes WER (Word Error Rate) and CER (Character Error Rate) via `jiwer`
- Supports oracle analysis: "What if we had picked the best ASR per segment?"
- Filters by decision (accept/human/reject) for per-category WER

**Benchmarking (`bench.py`):**
- Runs full pipeline on test suites (HuggingFace datasets)
- Compares model variations (e.g., "baseline" vs. "with CTC verification")
- Generates summary stats (WER by language, decision distribution)

---

### 10. Dependency Map

```
pyproject.toml [project.optional-dependencies]
├── asr: faster-whisper, soundfile
├── torch: torch, torchaudio, transformers (LID, ASR, CTC)
├── gemini: google-genai
├── db: psycopg2, supabase
├── bench: datasets, huggingface_hub
├── full: all of above
└── base: boto3, jiwer, numpy, python-dotenv, rich, typer
```

---

### 11. Example Workflow

```bash
# 1. Setup
pip install -e '.[full]'
export R2_ENDPOINT_URL=... R2_ACCESS_KEY_ID=... R2_SECRET_ACCESS_KEY=...

# 2. Download
maya-transcribe download --video-id xyz --out data/

# 3. Run pipeline
maya-transcribe run \
  --segments-dir data/xyz/segments \
  --out outputs/xyz \
  --whisper-model large-v3 \
  --transformers-whisper-model facebook/indic_whisper_v1 \
  --ctc-model facebook/wav2vec2-indic-1billion \
  --gemini-model gemini-2.0-flash

# 4. Evaluate
maya-transcribe eval \
  --results outputs/xyz/results.jsonl \
  --refs gold_transcripts.jsonl

# 5. Review failures
# Output: outputs/xyz/results.jsonl contains "human" decisions
# Review audio + top hypotheses + disagreement spans for quick correction
```

---

### 12. File Statistics

| Category | Count | Total Lines |
|----------|-------|------------|
| Core pipeline | 5 files | ~1800 |
| ASR/LID backends | 5 files | ~650 |
| Verification/combination | 3 files | ~850 |
| Utilities (audio, align, lang, norm, ids) | 5 files | ~520 |
| CLI & I/O | 3 files | ~500 |
| Optional (Gemini, eval, bench, DB, R2) | 6 files | ~800 |
| **Total** | **22 Python files** | **~6900 lines** |

---

## Summary

**Maya-Transcribe** is a quality-first, offline transcription pipeline for cleaned Indic & English segments. It prioritizes:

1. **Language routing** via MMS-LID (256 languages)
2. **Diverse ASR ensemble** (Whisper + IndicWhisper + IndicConformer + optional Gemini)
3. **System combination** via token-level ROVER voting
4. **Audio-grounded verification** via CTC loss + forced alignment
5. **Conservative acceptance** (flag uncertain/ambiguous for human review)
6. **Full auditability** (stores all hypotheses, disagreements, scores)

The codebase is modular, well-tested on benchmarks, and extensible for new models/languages.