Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 260
Output:
     1	from __future__ import annotations
     2
     3	import json
     4	from pathlib import Path
     5
     6	import typer
     7	from dotenv import load_dotenv
     8
     9	from maya_transcribe.audio import compute_qc, ffprobe, normalize_audio
    10	from maya_transcribe.config import Settings
    11	from maya_transcribe.dataset import find_segments_dir
    12	from maya_transcribe.ids import segment_identity
    13	from maya_transcribe.io import write_jsonl
    14	from maya_transcribe.r2 import R2Client
    15	from maya_transcribe.tar import safe_extract_tar
    16
    17	load_dotenv(dotenv_path=Path(".env"), override=False)
    18
    19	app = typer.Typer(add_completion=False, no_args_is_help=True)
    20	bench_app = typer.Typer(add_completion=False, no_args_is_help=True, help="Benchmarking utilities.")
    21	app.add_typer(bench_app, name="bench")
    22
    23
    24	@app.command()
    25	def download(
    26	    video_id: str = typer.Option(..., help="YouTube video id (used to locate the tar in R2)."),
    27	    out: Path = typer.Option(Path("data"), help="Output directory."),
    28	    r2_key: str | None = typer.Option(None, help="Exact R2 object key (skips search)."),
    29	    bucket: str | None = typer.Option(None, help="R2 bucket override (defaults to env R2_BUCKET)."),
    30	    prefix: str | None = typer.Option(None, help="Optional R2 prefix override."),
    31	    use_db: bool = typer.Option(False, help="Fetch `r2_tar_key` from Supabase Postgres (DATABASE_URL)."),
    32	    dry_run: bool = typer.Option(False, help="Only show matched key(s); do not download."),
    33	) -> None:
    34	    """
    35	    Download the cleaned/segmented tar for a video from R2 and extract it.
    36	    """
    37	    settings = Settings.load()
    38	    r2 = R2Client.from_settings(settings)
    39	    if bucket:
    40	        r2 = R2Client(
    41	            endpoint_url=r2.endpoint_url,
    42	            access_key_id=r2.access_key_id,
    43	            secret_access_key=r2.secret_access_key,
    44	            bucket=bucket,
    45	        )
    46
    47	    out.mkdir(parents=True, exist_ok=True)
    48	    r2_prefix = prefix if prefix is not None else settings.r2_prefix
    49
    50	    matches: list[str]
    51	    if use_db and not r2_key:
    52	        from maya_transcribe.db import fetch_video_meta
    53
    54	        meta = fetch_video_meta(video_id)
    55	        if meta and meta.r2_tar_key:
    56	            r2_key = meta.r2_tar_key
    57
    58	    if r2_key:
    59	        matches = [r2_key]
    60	    else:
    61	        best = r2.find_video_tar_key(video_id=video_id, prefix=r2_prefix)
    62	        if not best:
    63	            typer.echo("No matching objects found in R2.")
    64	            raise typer.Exit(code=2)
    65	        matches = [best]
    66
    67	    if dry_run:
    68	        for key in matches:
    69	            typer.echo(key)
    70	        return
    71
    72	    key = matches[0]
    73	    tar_path = r2.download_to(key=key, out_dir=out / video_id)
    74	    extract_dir = out / video_id
    75	    safe_extract_tar(tar_path, extract_dir=extract_dir)
    76	    segments_dir = find_segments_dir(extract_dir)
    77	    typer.echo(f"Downloaded: {tar_path}")
    78	    typer.echo(f"Extracted to: {extract_dir}")
    79	    if segments_dir:
    80	        typer.echo(f"Segments dir: {segments_dir}")
    81
    82
    83	@app.command()
    84	def preprocess(
    85	    segments_dir: Path = typer.Option(..., exists=True, file_okay=False, help="Directory containing audio segments."),
    86	    out: Path = typer.Option(Path("outputs/preprocessed"), help="Output directory for normalized audio + QC."),
    87	    sample_rate: int = typer.Option(16000, help="Target sample rate."),
    88	    trim_silence: bool = typer.Option(True, help="Trim leading/trailing silence (ffmpeg silenceremove)."),
    89	    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
    90	    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
    91	) -> None:
    92	    """
    93	    Normalize audio to 16kHz mono WAV, optionally trim silence, and write QC metrics.
    94	    """
    95	    out.mkdir(parents=True, exist_ok=True)
    96
    97	    audio_files: list[Path] = []
    98	    for ext in ("*.wav", "*.mp3", "*.m4a", "*.flac", "*.ogg", "*.opus"):
    99	        audio_files.extend(sorted(segments_dir.rglob(ext)))
   100	    audio_files = sorted(set(audio_files))
   101
   102	    if not audio_files:
   103	        typer.echo(f"No audio files found under: {segments_dir}")
   104	        raise typer.Exit(code=2)
   105
   106	    rows = []
   107	    norm_dir = out / "audio"
   108	    for src in audio_files:
   109	        ident = segment_identity(src, root=segments_dir)
   110	        segment_id = ident.segment_id
   111	        dst = norm_dir / f"{segment_id}.wav"
   112	        normal…2843 chars truncated…a_transcribe.asr import FasterWhisperASR
   180
   181	    asr_backend = FasterWhisperASR(model_size_or_path=model)
   182	    res = asr_backend.transcribe(
   183	        audio_path=audio, language=language, beam_size=beam_size, best_of=best_of, temperature=temperature
   184	    )
   185	    typer.echo(res.text)
   186
   187
   188	@app.command()
   189	def gemini(
   190	    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
   191	    language: str | None = typer.Option(None, help="Language hint (e.g., hin/tam/eng)."),
   192	    model: str = typer.Option("gemini-2.0-flash", help="Gemini model name."),
   193	    prompt_file: Path | None = typer.Option(Path("Prompt.md"), help="Prompt file (defaults to Prompt.md)."),
   194	    max_output_tokens: int = typer.Option(512, help="Max output tokens."),
   195	) -> None:
   196	    """
   197	    Transcribe a single file with Gemini (audio-native LLM). Output is parsed/validated strictly.
   198	    """
   199	    from maya_transcribe.gemini_asr import GeminiASR, GeminiConfig, load_prompt_template
   200
   201	    prompt = load_prompt_template(prompt_file)
   202	    backend = GeminiASR(config=GeminiConfig(model=model, max_output_tokens=max_output_tokens), prompt_template=prompt)
   203	    res = backend.transcribe(audio_path=audio, language=language)
   204	    typer.echo(res.text)
   205
   206
   207	@app.command()
   208	def run(
   209	    segments_dir: Path = typer.Option(..., exists=True, file_okay=False, help="Directory containing audio segments."),
   210	    out: Path = typer.Option(Path("outputs/run"), help="Output directory."),
   211	    whisper_model: str = typer.Option("large-v3", help="Whisper model size/path for faster-whisper."),
   212	    lid_model_id: str = typer.Option("facebook/mms-lid-256", help="HuggingFace MMS-LID model id."),
   213	    require_lid_backend: bool = typer.Option(True, help="Fail if MMS-LID backend can't initialize."),
   214	    transformers_whisper_model: list[str] = typer.Option(
   215	        [], help="Extra whisper-family HF models to run via transformers (repeatable)."
   216	    ),
   217	    gemini_model: str | None = typer.Option(None, help="Optional Gemini model name (adds Gemini as a hypothesis)."),
   218	    gemini_prompt_file: Path | None = typer.Option(Path("Prompt.md"), help="Prompt file for Gemini."),
   219	    gemini_hard_only: bool = typer.Option(True, help="Only call Gemini on 'hard' segments."),
   220	    ctc_model: str | None = typer.Option(None, help="Optional HF CTC model id for audio-grounded fit scoring."),
   221	    ctc_model_map: Path | None = typer.Option(None, help="JSON map {lang_code: hf_model_id} for CTC scoring."),
   222	    require_ctc: bool = typer.Option(True, help="When CTC is enabled, require a CTC score to accept."),
   223	    min_ctc_margin: float = typer.Option(0.05, help="Min (second-best - best) CTC loss margin to auto-accept."),
   224	    trim_silence: bool = typer.Option(True, help="Trim leading/trailing silence (ffmpeg silenceremove)."),
   225	    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
   226	    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
   227	    limit: int | None = typer.Option(None, help="Process only the first N segments."),
   228	    thresholds_json: Path | None = typer.Option(None, help="Optional JSON file of threshold overrides (verify + routing)."),
   229	) -> None:
   230	    """
   231	    End-to-end pipeline: preprocess -> LID -> ASR ensemble -> ROVER -> verify.
   232	    """
   233	    from maya_transcribe.pipeline import run_pipeline
   234
   235	    ctc_map = None
   236	    if ctc_model_map is not None:
   237	        ctc_map = json.loads(ctc_model_map.read_text(encoding="utf-8"))
   238	        if not isinstance(ctc_map, dict):
   239	            raise typer.BadParameter("ctc_model_map must be a JSON object {lang_code: model_id}")
   240
   241	    thresholds: dict[str, object] = {}
   242	    if thresholds_json is not None:
   243	        loaded = json.loads(thresholds_json.read_text(encoding="utf-8"))
   244	        if not isinstance(loaded, dict):
   245	            raise typer.BadParameter("thresholds_json must be a JSON object")
   246	        thresholds = loaded
   247	    if ctc_model or ctc_map:
   248	        thresholds["require_ctc"] = require_ctc
   249	        thresholds["min_ctc_margin"] = float(min_ctc_margin)
   250
   251	    results_path = run_pipeline(
   252	        segments_dir=segments_dir,
   253	        out_dir=out,
   254	        lid_model_id=lid_model_id,
   255	        require_lid_backend=require_lid_backend,
   256	        whisper_model=whisper_model,
   257	        extra_transformers_whisper_models=transformers_whisper_model,
   258	        gemini_model=gemini_model,
   259	        gemini_prompt_path=gemini_prompt_file,
   260	        gemini_hard_only=gemini_hard_only,


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


set -euo pipefail
nl -ba maya_transcribe/cli.py | sed -n '260,520p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   260	        gemini_hard_only=gemini_hard_only,
   261	        ctc_model=ctc_model,
   262	        ctc_model_map=ctc_map,
   263	        trim_silence=trim_silence,
   264	        silence_threshold_db=silence_threshold_db,
   265	        min_silence_sec=min_silence_sec,
   266	        limit=limit,
   267	        thresholds=thresholds,
   268	    )
   269	    typer.echo(f"Wrote: {results_path}")
   270	    typer.echo(f"Wrote: {out / 'summary.json'}")
   271
   272
   273	@app.command()
   274	def eval(
   275	    results: Path = typer.Option(..., exists=True, dir_okay=False, help="Pipeline results JSONL (outputs/run/results.jsonl)."),
   276	    refs: Path = typer.Option(..., exists=True, dir_okay=False, help="Reference transcripts JSONL."),
   277	    out: Path | None = typer.Option(None, help="Write summary JSON to this path."),
   278	    hypothesis: str = typer.Option('final', help='Hypothesis to evaluate: "final", "rover", or "asr:<key>".'),
   279	    oracle: bool = typer.Option(True, help="Compute oracle-best and selection-gap stats across all candidates."),
   280	    language_source: str = typer.Option("route", help="Use language from: 'route' (LID) or 'refs' (refs.jsonl)."),
   281	) -> None:
   282	    """
   283	    Compute WER/CER for a hypothesis against a reference JSONL keyed by `segment_id`.
   284	    """
   285	    from maya_transcribe.eval import evaluate
   286
   287	    summary = evaluate(
   288	        results_path=results,
   289	        refs_path=refs,
   290	        out_path=out,
   291	        hypothesis=hypothesis,
   292	        oracle=oracle,
   293	        language_source=language_source,
   294	    )
   295	    typer.echo(json.dumps(summary, ensure_ascii=False, indent=2))
   296
   297
   298	@bench_app.command("build")
   299	def bench_build(
   300	    out: Path = typer.Option(Path("data/bench/maya_hf_v0"), help="Output benchmark corpus directory."),
   301	    samples_per_lang: int = typer.Option(200, help="Target samples per language."),
   302	    seed: int = typer.Option(0, help="Shuffle seed."),
   303	    shuffle: bool = typer.Option(True, help="Shuffle within each source before selecting."),
   304	    lang: list[str] = typer.Option([], help="Limit to ISO-639-3 language codes (repeatable)."),
   305	    min_duration_sec: float = typer.Option(1.0, help="Min audio duration."),
   306	    max_duration_sec: float = typer.Option(30.0, help="Max audio duration."),
   307	    force: bool = typer.Option(False, help="Delete and rebuild output directory if it exists."),
   308	) -> None:
   309	    """
   310	    Build a local benchmark corpus from Hugging Face datasets.
   311	    """
   312	    from maya_transcribe.bench import build_benchmark_corpus
   313
   314	    manifest = build_benchmark_corpus(
   315	        out_dir=out,
   316	        samples_per_lang=samples_per_lang,
   317	        seed=seed,
   318	        shuffle=shuffle,
   319	        langs=lang or None,
   320	        min_duration_sec=min_duration_sec,
   321	        max_duration_sec=max_duration_sec,
   322	        force=force,
   323	    )
   324	    typer.echo(json.dumps({"wrote": str(out), "total_rows": manifest.get("total_rows"), "counts": manifest.get("counts")}, indent=2))
   325
   326
   327	@bench_app.command("run")
   328	def bench_run(
   329	    bench_dir: Path = typer.Option(Path("data/bench/maya_hf_v0"), exists=True, file_okay=False, help="Benchmark corpus directory."),
   330	    out: Path | None = typer.Option(None, help="Output directory (defaults under outputs/bench/)."),
   331	    whisper_model: str = typer.Option("large-v3", help="Whisper model size/path for faster-whisper."),
   332	    lid_model_id: str = typer.Option("facebook/mms-lid-256", help="HuggingFace MMS-LID model id."),
   333	    require_lid_backend: bool = typer.Option(True, help="Fail if MMS-LID backend can't initialize."),
   334	    transformers_whisper_model: list[str] = typer.Option(
   335	        [], help="Extra whisper-family HF models to run via transformers (repeatable)."
   336	    ),
   337	    ctc_model: str | None = typer.Option(None, help="Optional HF CTC model id for audio-grounded fit scoring."),
   338	    ctc_model_map: Path | None = typer.Option(None, help="JSON map {lang_code: hf_model_id} for CTC scoring."),
   339	    require_ctc: bool = typer.Option(True, help="When CTC is enabled, require a CTC score to accept."),
   340	    min_ctc_margin: float = typer.Option(0.05, help="Min (second-best - best) CTC loss margin to auto-accept."),
   341	    trim_silence: bool = typer.Option(True, help="Trim leading/trailing silence (ffmpeg silenceremove)."),
   342	    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
   343	    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
   344	    limit: int | None = typer.Option(None, help="Process only the first N segments."),
   345	    thresholds_json: Path | None = typer.Option(None, help="Optional JSON file of threshold overrides (verify + routing)."),
   346	    hypothesis: str = typer.Option("final", help='Hypothesis to evaluate: "final", "rover", or "asr:<key>".'),
   347	) -> None:
   348	    """
   349	    Run the full pipeline on a local benchmark corpus and write eval.json.
   350	    """
   351	    from datetime import datetime
   352
   353	    from maya_transcribe.eval import evaluate
   354	    from maya_transcribe.pipeline import run_pipeline
   355
   356	    audio_dir = bench_dir / "audio"
   357	    refs_path = bench_dir / "refs.jsonl"
   358	    if not audio_dir.exists():
   359	        raise typer.BadParameter(f"Missing: {audio_dir}")
   360	    if not refs_path.exists():
   361	        raise typer.BadParameter(f"Missing: {refs_path}")
   362
   363	    if out is None:
   364	        ts = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
   365	        out = Path("outputs/bench") / bench_dir.name / f"run_{ts}"
   366
   367	    ctc_map = None
   368	    if ctc_model_map is not None:
   369	        ctc_map = json.loads(ctc_model_map.read_text(encoding="utf-8"))
   370	        if not isinstance(ctc_map, dict):
   371	            raise typer.BadParameter("ctc_model_map must be a JSON object {lang_code: model_id}")
   372
   373	    thresholds: dict[str, object] = {}
   374	    if thresholds_json is not None:
   375	        loaded = json.loads(thresholds_json.read_text(encoding="utf-8"))
   376	        if not isinstance(loaded, dict):
   377	            raise typer.BadParameter("thresholds_json must be a JSON object")
   378	        thresholds = loaded
   379	    if ctc_model or ctc_map:
   380	        thresholds["require_ctc"] = require_ctc
   381	        thresholds["min_ctc_margin"] = float(min_ctc_margin)
   382
   383	    results_path = run_pipeline(
   384	        segments_dir=audio_dir,
   385	        out_dir=out,
   386	        lid_model_id=lid_model_id,
   387	        require_lid_backend=require_lid_backend,
   388	        whisper_model=whisper_model,
   389	        extra_transformers_whisper_models=transformers_whisper_model,
   390	        ctc_model=ctc_model,
   391	        ctc_model_map=ctc_map,
   392	        trim_silence=trim_silence,
   393	        silence_threshold_db=silence_threshold_db,
   394	        min_silence_sec=min_silence_sec,
   395	        limit=limit,
   396	        thresholds=thresholds,
   397	    )
   398
   399	    eval_path = out / "eval.json"
   400	    summary = evaluate(
   401	        results_path=results_path,
   402	        refs_path=refs_path,
   403	        out_path=eval_path,
   404	        hypothesis=hypothesis,
   405	        oracle=True,
   406	        language_source="refs",
   407	    )
   408	    typer.echo(f"Wrote: {results_path}")
   409	    typer.echo(f"Wrote: {out / 'summary.json'}")
   410	    typer.echo(f"Wrote: {eval_path}")
   411	    typer.echo(json.dumps(summary, ensure_ascii=False, indent=2))
   412
   413
   414	@bench_app.command("push")
   415	def bench_push(
   416	    bench_dir: Path = typer.Option(Path("data/bench/maya_hf_v0"), exists=True, file_okay=False, help="Benchmark corpus directory."),
   417	    repo_id: str = typer.Option(..., help="Hugging Face dataset repo id (e.g., org/name)."),
   418	    private: bool = typer.Option(True, help="Create/push as a private dataset repo."),
   419	    include_audio: bool = typer.Option(False, help="Include audio bytes in the pushed dataset (check dataset licenses!)."),
   420	    split: str = typer.Option("test", help="Split name to use on the Hub."),
   421	) -> None:
   422	    """
   423	    Push a local benchmark corpus to the Hugging Face Hub as a (private) dataset.
   424	    """
   425	    from maya_transcribe.bench import push_benchmark_to_hub
   426
   427	    repo = push_benchmark_to_hub(bench_dir=bench_dir, repo_id=repo_id, private=private, include_audio=include_audio, split=split)
   428	    typer.echo(f"Pushed: {repo}")
   429
   430
   431	if __name__ == "__main__":
   432	    app()


Summary: auto


**Evaluating options for gemini**

I see that the command structure for bench build/run/push looks okay, but the run command doesn’t include the gemini options, possibly to avoid costs. It could be useful for the benchmarking to consider including gemini since it's seen as a superior model for other languages. I wonder if comparing with gemini might help establish an upper bound for evaluations, especially given the inclusion of thresholds_json for both run and bench_run. Now, I’ll check eval.py.


set -euo pipefail
nl -ba maya_transcribe/eval.py | sed -n '1,260p'


Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 260
Output:
     1	from __future__ import annotations
     2
     3	import json
     4	from dataclasses import dataclass
     5	from pathlib import Path
     6	from typing import Any
     7
     8	from jiwer import cer, wer
     9
    10	from maya_transcribe.io import read_jsonl
    11	from maya_transcribe.textnorm import normalize_text
    12
    13
    14	@dataclass(frozen=True)
    15	class EvalRow:
    16	    segment_id: str
    17	    language: str | None
    18	    decision: str | None
    19	    duration_sec: float | None
    20	    duration_bucket: str | None
    21	    ref: str
    22	    hyp: str
    23	    wer: float
    24	    cer: float
    25	    oracle_best_wer: float | None = None
    26	    oracle_best_cer: float | None = None
    27	    oracle_best_key: str | None = None
    28	    selection_gap_wer: float | None = None
    29
    30
    31	def load_refs(refs_path: Path) -> dict[str, dict[str, Any]]:
    32	    """
    33	    Read a JSONL file of references.
    34	    Expected keys (any one): `text`, `transcript`, `reference`.
    35	    Required key: `segment_id` (or `id`).
    36	    """
    37	    refs: dict[str, dict[str, Any]] = {}
    38	    for row in read_jsonl(refs_path):
    39	        segment_id = str(row.get("segment_id") or row.get("id") or "").strip()
    40	        if not segment_id:
    41	            continue
    42	        refs[segment_id] = row
    43	    return refs
    44
    45
    46	def _get_text(row: dict[str, Any], keys: tuple[str, ...]) -> str | None:
    47	    for k in keys:
    48	        v = row.get(k)
    49	        if isinstance(v, str) and v.strip():
    50	            return v
    51	    return None
    52
    53
    54	def evaluate(
    55	    results_path: Path,
    56	    refs_path: Path,
    57	    out_path: Path | None = None,
    58	    hypothesis: str = "final",
    59	    oracle: bool = True,
    60	    language_source: str = "route",
    61	) -> dict[str, Any]:
    62	    refs = load_refs(refs_path)
    63	    results = read_jsonl(results_path)
    64
    65	    rows: list[EvalRow] = []
    66	    for r in results:
    67	        segment_id = str(r.get("segment_id") or "").strip()
    68	        if not segment_id or segment_id not in refs:
    69	            continue
    70
    71	        ref_row = refs[segment_id]
    72	        ref_text = _get_text(ref_row, ("text", "transcript", "reference"))
    73	        if ref_text is None:
    74	            continue
    75
    76	        hyp_text = extract_hypothesis_text(r, hypothesis=hypothesis)
    77	        if hyp_text is None:
    78	            continue
    79
    80	        route_lang = None
    81	        route = r.get("route") or {}
    82	        if isinstance(route, dict):
    83	            route_lang = route.get("language_hint")
    84
    85	        ref_lang = None
    86	        if language_source == "refs":
    87	            v = ref_row.get("lang") or ref_row.get("language")
    88	            if isinstance(v, str) and v.strip():
    89	                ref_lang = v.strip()
    90	        elif language_source != "route":
    91	            raise ValueError("language_source must be 'route' or 'refs'")
    92
    93	        lang = ref_lang or route_lang
    94
    95	        decision = None
    96	        ver = r.get("verification") or {}
    97	        if isinstance(ver, dict):
    98	            decision = ver.get("decision")
    99
   100	        duration_sec = None
   101	        qc = r.get("qc") or {}
   102	        if isinstance(qc, dict):
   103	            try:
   104	                duration_sec = float(qc.get("duration_sec")) if qc.get("duration_sec") is not None else None
   105	            except (TypeError, ValueError):
   106	                duration_sec = None
   107	        bucket = duration_bucket(duration_sec)
   108
   109	        ref_norm = normalize_text(ref_text, lang=lang)
   110	        hyp_norm = normalize_text(hyp_text, lang=lang)
   111	        w = float(wer(ref_norm, hyp_norm))
   112	        c = float(cer(ref_norm.replace(" ", ""), hyp_norm.replace(" ", "")))
   113
   114	        oracle_best_wer = None
   115	        oracle_best_cer = None
   116	        oracle_best_key = None
   117	        selection_gap_wer = None
   118
   119	        if oracle:
   120	            cand_wer: dict[str, float] = {}
   121	            cand_cer: dict[str, float] = {}
   122	            for key, text in extract_all_candidates(result_row=r).items():
   123	                ref_n = ref_norm
   124	                hyp_n = normalize_text(text, lang=lang)
   125	                cand_wer[key] = float(wer(ref_n, hyp_n))
   126	                cand_cer[key] = float(cer(ref_n.replace(" ", ""), hyp_n.replace(" ", "")))
   127
   128	            if cand_wer:
   129	                oracle_best_key, oracle_best_wer = min(cand_wer.items(), key=lambda kv: kv[1])
   130	                oracle_best_cer = cand_cer.get(oracle_best_key)
   131	                selection_gap_wer = float(w) …1521 chars truncated…append(row.cer)
   166	        if row.duration_bucket:
   167	            by_bucket.setdefault(row.duration_bucket, []).append(row.wer)
   168	            by_bucket_cer.setdefault(row.duration_bucket, []).append(row.cer)
   169	        if row.selection_gap_wer is not None:
   170	            selection_gaps.append(float(row.selection_gap_wer))
   171	        if row.oracle_best_wer is not None:
   172	            oracle_best_wers.append(float(row.oracle_best_wer))
   173
   174	    summary = {
   175	        "hypothesis": hypothesis,
   176	        "pairs_evaluated": len(rows),
   177	        "wer_mean": (sum(r.wer for r in rows) / len(rows)) if rows else None,
   178	        "cer_mean": (sum(r.cer for r in rows) / len(rows)) if rows else None,
   179	        "wer_p50": _percentile([r.wer for r in rows], 0.50),
   180	        "wer_p90": _percentile([r.wer for r in rows], 0.90),
   181	        "wer_p95": _percentile([r.wer for r in rows], 0.95),
   182	        "cer_p50": _percentile([r.cer for r in rows], 0.50),
   183	        "cer_p90": _percentile([r.cer for r in rows], 0.90),
   184	        "cer_p95": _percentile([r.cer for r in rows], 0.95),
   185	        "segment_error_rate": (sum(1 for r in rows if r.wer > 0.0) / len(rows)) if rows else None,
   186	        "pct_wer_gt_5pct": (sum(1 for r in rows if r.wer > 0.05) / len(rows)) if rows else None,
   187	        "pct_wer_gt_10pct": (sum(1 for r in rows if r.wer > 0.10) / len(rows)) if rows else None,
   188	        "wer_by_language_mean": {k: sum(v) / len(v) for k, v in sorted(by_lang.items())},
   189	        "wer_by_decision_mean": {k: sum(v) / len(v) for k, v in sorted(by_dec.items())},
   190	        "wer_by_duration_bucket_mean": {k: sum(v) / len(v) for k, v in sorted(by_bucket.items())},
   191	        "cer_by_language_mean": {k: sum(v) / len(v) for k, v in sorted(by_lang_cer.items())},
   192	        "cer_by_decision_mean": {k: sum(v) / len(v) for k, v in sorted(by_dec_cer.items())},
   193	        "cer_by_duration_bucket_mean": {k: sum(v) / len(v) for k, v in sorted(by_bucket_cer.items())},
   194	        "oracle": {
   195	            "enabled": bool(oracle),
   196	            "oracle_best_wer_mean": (sum(oracle_best_wers) / len(oracle_best_wers)) if oracle_best_wers else None,
   197	            "selection_gap_wer_mean": (sum(selection_gaps) / len(selection_gaps)) if selection_gaps else None,
   198	            "selection_gap_wer_p90": _percentile(selection_gaps, 0.90),
   199	        },
   200	    }
   201
   202	    if out_path is not None:
   203	        out_path.parent.mkdir(parents=True, exist_ok=True)
   204	        out_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
   205
   206	    return summary
   207
   208
   209	def extract_hypothesis_text(result_row: dict[str, Any], hypothesis: str) -> str | None:
   210	    """
   211	    hypothesis:
   212	      - "final": uses result_row["final"]["text"] (falls back to rover if missing)
   213	      - "rover" (default): uses result_row["rover"]["text"]
   214	      - "asr:<key>": uses result_row["asr"][<key>]["text"]
   215	    """
   216	    hypothesis = (hypothesis or "rover").strip()
   217	    if hypothesis == "final":
   218	        final = result_row.get("final") or {}
   219	        if isinstance(final, dict) and isinstance(final.get("text"), str):
   220	            return final["text"]
   221	        hypothesis = "rover"
   222	    if hypothesis == "rover":
   223	        rover = result_row.get("rover") or {}
   224	        if isinstance(rover, dict) and isinstance(rover.get("text"), str):
   225	            return rover["text"]
   226	        return None
   227
   228	    if hypothesis.startswith("asr:"):
   229	        key = hypothesis[len("asr:") :].strip()
   230	        asr = result_row.get("asr") or {}
   231	        if isinstance(asr, dict):
   232	            entry = asr.get(key) or {}
   233	            if isinstance(entry, dict) and isinstance(entry.get("text"), str):
   234	                return entry["text"]
   235	        return None
   236
   237	    return None
   238
   239
   240	def extract_all_candidates(result_row: dict[str, Any]) -> dict[str, str]:
   241	    """
   242	    Collect all text candidates available in a pipeline result row.
   243	    Keys are stable and namespaced (final/rover/asr:<key>).
   244	    """
   245	    candidates: dict[str, str] = {}
   246
   247	    final = extract_hypothesis_text(result_row, "final")
   248	    if final is not None:
   249	        candidates["final"] = final
   250
   251	    rover = extract_hypothesis_text(result_row, "rover")
   252	    if rover is not None:
   253	        candidates["rover"] = rover
   254
   255	    asr = result_row.get("asr") or {}
   256	    if isinstance(asr, dict):
   257	        for key, entry in asr.items():
   258	            if not isinstance(key, str):
   259	                continue
   260	            if not isinstance(entry, dict):


set -euo pipefail
nl -ba maya_transcribe/eval.py | sed -n '260,420p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   260	            if not isinstance(entry, dict):
   261	                continue
   262	            text = entry.get("text")
   263	            if isinstance(text, str) and text.strip():
   264	                candidates[f"asr:{key}"] = text
   265
   266	    # Also include normalized hypotheses if present (useful for debugging).
   267	    normalized = result_row.get("normalized") or {}
   268	    if isinstance(normalized, dict):
   269	        for key, text in normalized.items():
   270	            if not isinstance(key, str) or not isinstance(text, str):
   271	                continue
   272	            if text.strip():
   273	                candidates[f"norm:{key}"] = text
   274
   275	    return candidates
   276
   277
   278	def duration_bucket(duration_sec: float | None) -> str | None:
   279	    if duration_sec is None:
   280	        return None
   281	    if duration_sec < 3.0:
   282	        return "0-3s"
   283	    if duration_sec < 10.0:
   284	        return "3-10s"
   285	    if duration_sec < 30.0:
   286	        return "10-30s"
   287	    if duration_sec < 60.0:
   288	        return "30-60s"
   289	    return "60s+"
   290
   291
   292	def _percentile(xs: list[float], q: float) -> float | None:
   293	    if not xs:
   294	        return None
   295	    xs_sorted = sorted(xs)
   296	    idx = int(round((len(xs_sorted) - 1) * q))
   297	    idx = max(0, min(len(xs_sorted) - 1, idx))
   298	    return float(xs_sorted[idx])


set -euo pipefail
nl -ba maya_transcribe/textnorm.py | sed -n '1,240p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	import unicodedata
     4
     5
     6	_ZERO_WIDTH = {
     7	    "\u200b",  # zero width space
     8	    "\u200c",  # zero width non-joiner
     9	    "\u200d",  # zero width joiner
    10	    "\ufeff",  # byte order mark
    11	}
    12
    13
    14	def normalize_text(text: str, lang: str | None = None) -> str:
    15	    """
    16	    Conservative normalization for hypothesis alignment/comparison (not for publishing).
    17	    - Unicode NFKC
    18	    - Drop punctuation
    19	    - Normalize all Unicode digits to ASCII
    20	    - Collapse whitespace
    21	    - Lowercase for English only
    22	    """
    23	    if not text:
    24	        return ""
    25
    26	    text = unicodedata.normalize("NFKC", text)
    27
    28	    out_chars: list[str] = []
    29	    for ch in text:
    30	        if ch in _ZERO_WIDTH:
    31	            continue
    32
    33	        cat = unicodedata.category(ch)
    34	        if cat.startswith("P"):
    35	            continue
    36
    37	        if cat == "Zs" or ch in ("\t", "\n", "\r"):
    38	            out_chars.append(" ")
    39	            continue
    40
    41	        if cat == "Nd":
    42	            try:
    43	                out_chars.append(str(unicodedata.digit(ch)))
    44	                continue
    45	            except (TypeError, ValueError):
    46	                pass
    47
    48	        out_chars.append(ch)
    49
    50	    norm = "".join(out_chars)
    51	    norm = " ".join(norm.split())
    52
    53	    if (lang or "").lower() in {"en", "eng"}:
    54	        norm = norm.lower()
    55
    56	    return norm
    57
    58
    59	def tokenize_words(text: str) -> list[str]:
    60	    text = " ".join((text or "").split())
    61	    if not text:
    62	        return []
    63	    return text.split(" ")
    64


**Evaluating transcript quality**

It looks like the normalize_text function is conservative, which is good. If there's a high Word Error Rate (WER), it might mean that transcripts are very different from the references or maybe even lacking altogether. This could happen if the pipeline doesn’t accept transcripts, uses `<unk>`, or ends up outputting blanks. I could consider how the language identification (LID) process impacts the automatic speech recognition (ASR) system too. It seems there’s a complex interaction happening!


**Analyzing WER calculations**

It sounds like they might want to measure Word Error Rate (WER) specifically for accepted segments, which makes sense! Computing WER by decision group could show that while overall WER is high, accepted segments might still be performing well. They need to take that into account. It seems like there are concerns around language identification (LID) outputs and potential failures when mapping languages. The fixes they've implemented suggest ongoing improvements for handling language codes efficiently. I’m curious how these changes will affect performance!


**Verifying code robustness**

It’s looking good, but I need to make sure it’s actually correct and robust. I should inspect the `ctc_fit.py` file, particularly the lines around 1 to 140, to confirm everything is functioning as intended. This step seems crucial to ensure that whatever changes or implementations were made are solid and won’t lead to unexpected issues down the line. I’m curious about what I might find in those lines!


set -euo pipefail
nl -ba maya_transcribe/ctc_fit.py | sed -n '1,220p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	from dataclasses import dataclass
     4	from pathlib import Path
     5	from typing import Any
     6
     7	from maya_transcribe.audio import decode_pcm16
     8	from maya_transcribe.language import ISO3_TO_WHISPER_LANG
     9	from maya_transcribe.textnorm import normalize_text
    10
    11
    12	@dataclass(frozen=True)
    13	class CTCScore:
    14	    model_id: str
    15	    loss: float
    16	    text: str
    17	    unk_ratio: float | None
    18	    target_len: int
    19	    token_score_mean: float | None
    20	    token_score_min: float | None
    21	    token_coverage: float | None
    22	    token_logp_threshold: float | None
    23	    span_count: int | None
    24	    span_mismatch: bool | None
    25
    26
    27	_WHISPER_TO_ISO3: dict[str, str] = {v: k for k, v in ISO3_TO_WHISPER_LANG.items()}
    28
    29
    30	def _to_iso3(lang: str | None) -> str | None:
    31	    if not lang:
    32	        return None
    33	    code = lang.strip().lower()
    34	    if not code:
    35	        return None
    36	    # Handle common BCP-47 forms (e.g., hi_IN / hi-IN).
    37	    code = code.replace("-", "_")
    38	    if "_" in code:
    39	        code = code.split("_", 1)[0]
    40	    if len(code) == 3:
    41	        return code
    42	    if len(code) == 2:
    43	        return _WHISPER_TO_ISO3.get(code, None)
    44	    return code
    45
    46
    47	class _CTCModel:
    48	    def __init__(self, model_id: str, device: str | None = None) -> None:
    49	        try:
    50	            import torch
    51	            from transformers import AutoModelForCTC, AutoProcessor
    52	        except Exception as e:  # pragma: no cover
    53	            raise RuntimeError(
    54	                "CTC scoring requires torch + transformers. Install with: pip install -e '.[torch]'"
    55	            ) from e
    56
    57	        if device is None:
    58	            device = "cuda" if torch.cuda.is_available() else "cpu"
    59
    60	        self.model_id = model_id
    61	        self.device = device
    62	        self._torch = torch
    63	        self._processor = AutoProcessor.from_pretrained(model_id)
    64	        self._model = AutoModelForCTC.from_pretrained(model_id).to(device)
    65	        self._model.eval()
    66
    67	        tokenizer = getattr(self._processor, "tokenizer", None)
    68	        self._force_uppercase = False
    69	        if tokenizer is not None and hasattr(tokenizer, "get_vocab"):
    70	            vocab = tokenizer.get_vocab()
    71	            # Common for wav2vec2 CTC: vocab contains A-Z but not a-z; transcripts should be uppercased.
    72	            self._force_uppercase = ("A" in vocab) and ("a" not in vocab)
    73
    74	        blank = getattr(self._model.config, "pad_token_id", None)
    75	        if blank is None and getattr(self._processor, "tokenizer", None) is not None:
    76	            blank = getattr(self._processor.tokenizer, "pad_token_id", None)
    77	        self.blank_id = int(blank) if blank is not None else 0
    78
    79	        self._target_lang: str | None = None
    80
    81	    def _maybe_set_target_lang(self, lang: str | None) -> None:
    82	        tokenizer = getattr(self._processor, "tokenizer", None)
    83	        set_lang = getattr(tokenizer, "set_target_lang", None) if tokenizer is not None else None
    84	        if not callable(set_lang):
    85	            return
    86
    87	        iso3 = _to_iso3(lang)
    88	        if not iso3:
    89	            return
    90	        if self._target_lang == iso3:
    91	            return
    92	        try:
    93	            set_lang(iso3)
    94	        except Exception as e:
    95	            raise RuntimeError(f"set_target_lang_failed:{iso3}:{e}") from e
    96	        self._target_lang = iso3
    97
    98	    def _compute_log_probs(self, audio_path: Path):
    99	        wav, sr = decode_pcm16(audio_path, sample_rate=16000, mono=True)
   100	        if wav.size == 0:
   101	            raise ValueError("empty_audio")
   102
   103	        inputs = self._processor(wav, sampling_rate=sr, return_tensors="pt")
   104
   105	        # Common key names: input_values (wav2vec2) or input_features (some processors).
   106	        if "input_values" in inputs:
   107	            model_inputs = {"input_values": inputs["input_values"].to(self.device)}
   108	        elif "input_features" in inputs:
   109	            model_inputs = {"input_features": inputs["input_features"].to(self.device)}
   110	        else:
   111	            # Best-effort: move all tensors to device.
   112	            model_inputs = {k: v.to(self.device) for k, v in inputs.items() if hasattr(v, "to")}
   113
   114	        with self._torch.no_grad():
   115	            logits = self._model(**model_inputs).logits  # (B,T,C)
   116	            log_probs = logits.log_softmax(dim=-1)  # (B,T,C)
   117
   118	        input_lengths = self._torch.tensor([log_probs.size(1)], dtype=self._torch.long, device=log_probs.device)
   119	        return log_probs, input_lengths
   120
   121	    def _score_from_log_probs(
   122	        self,
   123	        *,
   124	        log_probs,
   125	        input_lengths,
   126	        text: str,
   127	        lang: str | None,
   128	        max_unk_ratio: float,
   129	        token_logp_threshold: float,
   130	    ) -> CTCScore:
   131	        import torch.nn.functional as F
   132	        import warnings
   133
   134	        if not text:
   135	            raise ValueError("empty_text")
   136	        if "<unk>" in text or "<inaudible>" in text:
   137	            raise ValueError("contains_unk")
   138
   139	        # Conservative normalization helps tokenizer compatibility.
   140	        text_norm = normalize_text(text, lang=lang)
   141	        if not text_norm:
   142	            raise ValueError("empty_text")
   143	        if self._force_uppercase:
   144	            text_norm = text_norm.upper()
   145
   146	        # Tokenize targets (MMS multilingual tokenizers require set_target_lang()).
   147	        tokenizer = getattr(self._processor, "tokenizer", None)
   148	        if tokenizer is None:
   149	            raise ValueError("missing_tokenizer")
   150	        self._maybe_set_target_lang(lang)
   151	        tokenized = tokenizer(text_norm, add_special_tokens=False)
   152	        target_ids = tokenized.get("input_ids")
   153	        if not target_ids:
   154	            raise ValueError("empty_targets")
   155
   156	        unk_id = getattr(tokenizer, "unk_token_id", None)
   157	        unk_ratio: float | None = None
   158	        if unk_id is not None:
   159	            unk_count = sum(1 for tid in target_ids if int(tid) == int(unk_id))
   160	            unk_ratio = float(unk_count) / float(len(target_ids))
   161	            if unk_ratio > max_unk_ratio:
   162	                raise ValueError(f"too_many_unk_tokens:{unk_ratio:.3f}")
   163
   164	        # torch ctc_loss expects (T,N,C)
   165	        log_probs_tnc = log_probs.transpose(0, 1)
   166	        device = log_probs_tnc.device
   167	        targets = self._torch.tensor(target_ids, dtype=self._torch.long, device=device)
   168	        target_lengths = self._torch.tensor([targets.numel()], dtype=self._torch.long, device=device)
   169
   170	        # Reduction=mean normalizes by target length; good for comparability.
   171	        loss = F.ctc_loss(
   172	            log_probs_tnc,
   173	            targets,
   174	            input_lengths,
   175	            target_lengths,
   176	            blank=self.blank_id,
   177	            reduction="mean",
   178	            zero_infinity=True,
   179	        )
   180	        loss_value = float(loss.detach().cpu().item())
   181
   182	        # Forced alignment token scores (coverage/fit). Best-effort: alignment may fail for short audio vs long text.
   183	        token_score_mean = None
   184	        token_score_min = None
   185	        token_coverage = None
   186	        span_count = None
   187	        span_mismatch = None
   188
   189	        try:
   190	            import torchaudio
   191
   192	            with warnings.catch_warnings():
   193	                warnings.filterwarnings("ignore", message=".*forced_align.*deprecated.*", category=UserWarning)
   194	                aligned_tokens, aligned_scores = torchaudio.functional.forced_align(
   195	                    log_probs,
   196	                    targets.unsqueeze(0),
   197	                    input_lengths=input_lengths,
   198	                    target_lengths=target_lengths,
   199	                    blank=self.blank_id,
   200	                )
   201
   202	            spans = torchaudio.functional.merge_tokens(
   203	                aligned_tokens[0].detach().cpu(), aligned_scores[0].detach().cpu(), blank=self.blank_id
   204	            )
   205	            span_scores = [float(s.score) for s in spans]
   206	            span_count = len(span_scores)
   207	            span_mismatch = (span_count != int(target_lengths.item()))
   208	            if span_scores:
   209	                token_score_mean = sum(span_scores) / float(len(span_scores))
   210	                token_score_min = min(span_scores)
   211	                token_coverage = sum(1 for s in span_scores if s >= token_logp_threshold) / float(len(span_scores))
   212	        except Exception:
   213	            pass
   214
   215	        return CTCScore(
   216	            model_id=self.model_id,
   217	            loss=loss_value,
   218	            text=text,
   219	            unk_ratio=unk_ratio,
   220	            target_len=int(target_lengths.item()),


set -euo pipefail
nl -ba maya_transcribe/ctc_fit.py | sed -n '220,420p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   220	            target_len=int(target_lengths.item()),
   221	            token_score_mean=token_score_mean,
   222	            token_score_min=token_score_min,
   223	            token_coverage=token_coverage,
   224	            token_logp_threshold=token_logp_threshold,
   225	            span_count=span_count,
   226	            span_mismatch=span_mismatch,
   227	        )
   228
   229	    def score(
   230	        self,
   231	        audio_path: Path,
   232	        text: str,
   233	        lang: str | None = None,
   234	        *,
   235	        max_unk_ratio: float = 0.2,
   236	        token_logp_threshold: float = -3.0,
   237	    ) -> CTCScore:
   238	        log_probs, input_lengths = self._compute_log_probs(audio_path)
   239	        return self._score_from_log_probs(
   240	            log_probs=log_probs,
   241	            input_lengths=input_lengths,
   242	            text=text,
   243	            lang=lang,
   244	            max_unk_ratio=max_unk_ratio,
   245	            token_logp_threshold=token_logp_threshold,
   246	        )
   247
   248	    def score_candidates(
   249	        self,
   250	        *,
   251	        audio_path: Path,
   252	        candidates: dict[str, str],
   253	        lang: str | None,
   254	        max_unk_ratio: float,
   255	        token_logp_threshold: float,
   256	    ) -> tuple[list[tuple[str, CTCScore]], dict[str, str]]:
   257	        log_probs, input_lengths = self._compute_log_probs(audio_path)
   258
   259	        scored: list[tuple[str, CTCScore]] = []
   260	        errors: dict[str, str] = {}
   261
   262	        for key, text in candidates.items():
   263	            text = (text or "").strip()
   264	            if not text:
   265	                continue
   266	            try:
   267	                score = self._score_from_log_probs(
   268	                    log_probs=log_probs,
   269	                    input_lengths=input_lengths,
   270	                    text=text,
   271	                    lang=lang,
   272	                    max_unk_ratio=max_unk_ratio,
   273	                    token_logp_threshold=token_logp_threshold,
   274	                )
   275	                scored.append((key, score))
   276	            except Exception as e:
   277	                errors[key] = f"{type(e).__name__}:{e}"
   278
   279	        return scored, errors
   280
   281
   282	class CTCFitScorer:
   283	    """
   284	    Cache-loaded CTC scoring models for audio-grounded transcript fit.
   285	    """
   286
   287	    def __init__(
   288	        self,
   289	        model_map: dict[str, str] | None = None,
   290	        default_model: str | None = None,
   291	        *,
   292	        max_unk_ratio: float = 0.2,
   293	        token_logp_threshold: float = -3.0,
   294	    ) -> None:
   295	        self.model_map = model_map or {}
   296	        self.default_model = default_model
   297	        self.max_unk_ratio = float(max_unk_ratio)
   298	        self.token_logp_threshold = float(token_logp_threshold)
   299	        self._cache: dict[str, _CTCModel] = {}
   300
   301	    def _get_model_id(self, lang: str | None) -> str | None:
   302	        if lang and lang in self.model_map:
   303	            return self.model_map[lang]
   304	        return self.default_model
   305
   306	    def score(self, audio_path: Path, text: str, lang: str | None) -> CTCScore | None:
   307	        model_id = self._get_model_id(lang)
   308	        if not model_id:
   309	            return None
   310
   311	        model = self._cache.get(model_id)
   312	        if model is None:
   313	            model = _CTCModel(model_id=model_id)
   314	            self._cache[model_id] = model
   315
   316	        return model.score(
   317	            audio_path=audio_path,
   318	            text=text,
   319	            lang=lang,
   320	            max_unk_ratio=self.max_unk_ratio,
   321	            token_logp_threshold=self.token_logp_threshold,
   322	        )
   323
   324	    def score_candidates(
   325	        self, audio_path: Path, candidates: dict[str, str], lang: str | None
   326	    ) -> tuple[list[tuple[str, CTCScore]], dict[str, str]] | None:
   327	        model_id = self._get_model_id(lang)
   328	        if not model_id:
   329	            return None
   330
   331	        model = self._cache.get(model_id)
   332	        if model is None:
   333	            model = _CTCModel(model_id=model_id)
   334	            self._cache[model_id] = model
   335
   336	        return model.score_candidates(
   337	            audio_path=audio_path,
   338	            candidates=candidates,
   339	            lang=lang,
   340	            max_unk_ratio=self.max_unk_ratio,
   341	            token_logp_threshold=self.token_logp_threshold,
   342	        )
   343
   344
   345	def rank_candidates_by_ctc(
   346	    scorer: CTCFitScorer, audio_path: Path, candidates: dict[str, str], lang: str | None
   347	) -> dict[str, Any]:
   348	    scored: list[tuple[str, CTCScore]] = []
   349	    errors: dict[str, str] = {}
   350
   351	    batch = scorer.score_candidates(audio_path=audio_path, candidates=candidates, lang=lang)
   352	    if batch is not None:
   353	        scored, errors = batch
   354
   355	    scored.sort(key=lambda kv: kv[1].loss)
   356	    best = scored[0] if scored else None
   357
   358	    # Find the best *different-text* alternative, so duplicates don't create a fake "zero margin".
   359	    second = None
   360	    if best:
   361	        best_text = best[1].text
   362	        for item in scored[1:]:
   363	            if item[1].text != best_text:
   364	                second = item
   365	                break
   366
   367	    margin = (second[1].loss - best[1].loss) if (best and second) else None
   368
   369	    return {
   370	        "model_id": best[1].model_id if best else None,
   371	        "best_key": best[0] if best else None,
   372	        "best_loss": best[1].loss if best else None,
   373	        "second_best_key": second[0] if second else None,
   374	        "second_best_loss": second[1].loss if second else None,
   375	        "margin": float(margin) if margin is not None else None,
   376	        "scores": {
   377	            k: {
   378	                "loss": s.loss,
   379	                "model_id": s.model_id,
   380	                "unk_ratio": s.unk_ratio,
   381	                "target_len": s.target_len,
   382	                "token_score_mean": s.token_score_mean,
   383	                "token_score_min": s.token_score_min,
   384	                "token_coverage": s.token_coverage,
   385	                "token_logp_threshold": s.token_logp_threshold,
   386	                "span_count": s.span_count,
   387	                "span_mismatch": s.span_mismatch,
   388	            }
   389	            for k, s in scored
   390	        },
   391	        "errors": errors,
   392	    }


**Evaluating tensor shapes**

I’m noticing that ctc_fit.py has been updated to compute log probabilities and score multiple candidates. But I need to ensure that I understand how `tokenizer(text_norm, add_special_tokens=False)` works. It seems it returns a `BatchEncoding` with a list of integers, but some tokenizers may give a nested list.

If `target_ids` is a nested list, it might create a 2D tensor, while `ctc_loss` expects a 1D concatenated format. The docs suggest that `targets` can be either 1D or 2D, but `target_lengths` needs to match the batch size. I need to verify that!


**Analyzing tensor dimensions**

Alright, in forced alignment, they’re using `targets.unsqueeze(0)`. If `targets` is already 2D, this gives (1, 1, S), which doesn’t work. It looks like they expect `targets` to be 1D, so if the tokenized output is a nested list, it could lead to a bug. Typically, `tokenizer(text_norm, add_special_tokens=False)` for CTC tokenizers like `Wav2Vec2CTCTokenizer` returns a dictionary with 'input_ids' as a list of integers.

I think that should be okay, but I need to check how `merge_tokens` operates and what it returns!


**Reviewing token merging**

I'm recalling the function `torchaudio.functional.merge_tokens`, which merges consecutive identical tokens into spans. This process produce spans equal to the number of tokens, possibly including blanks that are removed. It seems this function is applied after `forced_align`, which aligns token IDs per frame, resulting in a compressed sequence by merging consecutive frames with the same token.

If target tokens repeat consecutively, they might not match the number of spans created, leading to fewer spans than expected. I need to ensure that any repeated tokens aren't incorrectly merged!


**Examining token mismatches**

When the target token sequence has repeated tokens, `forced_align` aligns them sequentially. However, because they’re identical, there might not be any blank frames between them, leading `merge_tokens` to merge them into one span. This could cause the span count to be less than the target length, which they interpret as a mismatch. This might be a bit too strict, especially since repeated tokens aren't that common in CTC char-level tokenization, but they can occur with double letters. I wonder if this mismatch detection is introducing unnecessary noise!


**Reviewing language code handling**

Now, I'm looking at how `_maybe_set_target_lang` handles ISO3 conversion from language codes. For the MMS model's tokenizer, it’s clear that `set_target_lang` expects ISO3 codes, like 'hin'. I'm wondering if it should accept both 'hin' and 'hi', but it seems the standard is to use the ISO3 code.

They convert BCP-47 forms, such as hi_IN to just 'hi'. I noted that they’re calling a `_WHISPER_TO_ISO3` mapping from `ISO3_TO_WHISPER_LANG`, which is defined in `language.py`. This mapping helps connect ISO3 codes to the Whisper language parameter codes. I’ll need to check how this all connects!


set -euo pipefail
nl -ba maya_transcribe/language.py | sed -n '1,240p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	ISO3_TO_WHISPER_LANG: dict[str, str] = {
     4	    "eng": "en",
     5	    "hin": "hi",
     6	    "mar": "mr",
     7	    "guj": "gu",
     8	    "pan": "pa",
     9	    "tam": "ta",
    10	    "tel": "te",
    11	    "kan": "kn",
    12	    "mal": "ml",
    13	    "ben": "bn",
    14	    "asm": "as",
    15	    "urd": "ur",
    16	    "san": "sa",
    17	}
    18
    19	# Human-friendly names and scripts (for prompts/UI).
    20	LANG_CODE_TO_NAME: dict[str, str] = {
    21	    "eng": "English",
    22	    "en": "English",
    23	    "hin": "Hindi",
    24	    "hi": "Hindi",
    25	    "mar": "Marathi",
    26	    "mr": "Marathi",
    27	    "guj": "Gujarati",
    28	    "gu": "Gujarati",
    29	    "pan": "Punjabi",
    30	    "pa": "Punjabi",
    31	    "tam": "Tamil",
    32	    "ta": "Tamil",
    33	    "tel": "Telugu",
    34	    "te": "Telugu",
    35	    "kan": "Kannada",
    36	    "kn": "Kannada",
    37	    "mal": "Malayalam",
    38	    "ml": "Malayalam",
    39	    "ben": "Bengali",
    40	    "bn": "Bengali",
    41	    "asm": "Assamese",
    42	    "as": "Assamese",
    43	    "ori": "Odia",
    44	    "ory": "Odia",
    45	    "or": "Odia",
    46	}
    47
    48	LANG_NAME_TO_SCRIPT: dict[str, str] = {
    49	    "English": "Latin",
    50	    "Hindi": "Devanagari",
    51	    "Marathi": "Devanagari",
    52	    "Gujarati": "Gujarati",
    53	    "Punjabi": "Gurmukhi",
    54	    "Tamil": "Tamil",
    55	    "Telugu": "Telugu",
    56	    "Kannada": "Kannada",
    57	    "Malayalam": "Malayalam",
    58	    "Bengali": "Bengali",
    59	    "Assamese": "Assamese",
    60	    "Odia": "Odia",
    61	}
    62
    63
    64	def to_whisper_lang(lang_code: str | None) -> str | None:
    65	    if not lang_code:
    66	        return None
    67	    lang_code = lang_code.strip().lower()
    68	    # Whisper/faster-whisper does not expose an Odia language token; let it auto-detect instead of erroring.
    69	    if lang_code in {"ori", "ory", "or"}:
    70	        return None
    71	    if len(lang_code) == 2:
    72	        return lang_code
    73	    return ISO3_TO_WHISPER_LANG.get(lang_code, None)
    74
    75
    76	def to_language_name(lang_code: str | None) -> str | None:
    77	    if not lang_code:
    78	        return None
    79	    lang_code = lang_code.strip().lower()
    80	    return LANG_CODE_TO_NAME.get(lang_code)
    81
    82
    83	def to_prompt_language_hint(lang_code: str | None) -> str:
    84	    """
    85	    Return a human-friendly language hint string for prompting audio-native models.
    86	    """
    87	    if not lang_code:
    88	        return "Unknown (auto-detect primary spoken language)"
    89	    lang_code_norm = lang_code.strip().lower()
    90	    name = LANG_CODE_TO_NAME.get(lang_code_norm)
    91	    if not name:
    92	        return f"{lang_code_norm} (auto-detect primary spoken language)"
    93	    script = LANG_NAME_TO_SCRIPT.get(name)
    94	    if script:
    95	        return f"{name} ({script} script)"
    96	    return name


**Considering language support**

I realized that faster-whisper has a list of languages, but it seems 'or' isn’t recognized as a valid code. The user switched to None, which raises a concern about language hinting for Odia segments and possible performance degradation with auto-detection. I wonder if treating Odia as Hindi might help, but it probably wouldn’t. Maybe IndicWhisper supports Odia, but I still need to check if using the token 'or' is valid. There’s a possibility that the transformer in `transformers` can use 'or' for its processes, but this remains unclear.


**Checking language codes**

I need to figure out if the language code 'or' for Oriya is included in Whisper's list in the Transformers library. I’ll quickly check using Python, though I wonder if it’s installed. From what I know, OpenAI Whisper supports it with the 'or' code, so it’s puzzling why faster-whisper says it’s invalid. Maybe it's because faster-whisper might use ISO 639-1 codes, or 'or' could conflict with something else. I should verify the installed version and check its language codes to clarify.


set -euo pipefail
python3 - <<'PY'
try:
    import faster_whisper
    from faster_whisper.transcribe import LANGUAGES
    print('faster_whisper', faster_whisper.__version__)
    print('has_or', 'or' in LANGUAGES)
    print('num_lang', len(LANGUAGES))
except Exception as e:
    print('error', e)
PY


Output:
Exit code: 0
Wall time: 2.4 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
error cannot import name 'LANGUAGES' from 'faster_whisper.transcribe' (/home/ubuntu/.local/lib/python3.10/site-packages/faster_whisper/transcribe.py)


Summary: auto


**Inspecting import error**

I’m facing an import error, specifically with the LANGUAGES attribute from the faster_whisper.transcribe module. I wonder if something changed in the latest version. So, I think the best course of action is to inspect the transcribe module and check what attributes are currently available. By printing the attributes, I can better figure out what’s happening and how I might resolve this issue. Let’s see what I find!


set -euo pipefail
python3 - <<'PY'
import faster_whisper
import faster_whisper.transcribe as t
print('faster_whisper', getattr(faster_whisper,'__version__',None))
print('attrs', [a for a in dir(t) if 'LANG' in a.upper()][:50])
PY


Output:
Exit code: 0
Wall time: 2.5 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
faster_whisper 1.2.1
attrs ['_LANGUAGE_CODES']


set -euo pipefail
python3 - <<'PY'
import faster_whisper.transcribe as t
codes = getattr(t,'_LANGUAGE_CODES',None)
print(type(codes), len(codes) if codes else None)
print('or' in codes if codes else None)
print('first10', list(codes)[:10])
PY


Output:
Exit code: 0
Wall time: 2.5 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
<class 'tuple'> 100
False
first10 ['af', 'am', 'ar', 'as', 'az', 'ba', 'be', 'bg', 'bn', 'bo']


set -euo pipefail
python3 - <<'PY'
try:
    import whisper
    print('whisper version', whisper.__version__ if hasattr(whisper,'__version__') else 'unknown')
    langs = whisper.tokenizer.LANGUAGES
    print('num', len(langs))
    print('or' in langs)
    if 'or' in langs:
        print('or->', langs['or'])
except Exception as e:
    print('error', e)
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
error No module named 'whisper'


set -euo pipefail
python3 - <<'PY'
import faster_whisper.transcribe as t
codes=set(t._LANGUAGE_CODES)
for code in ['ur','sa','hi','ta','te','kn','ml','bn','as','mr','gu','pa','en']:
    print(code, code in codes)
PY


Output:
Exit code: 0
Wall time: 2.3 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
ur True
sa True
hi True
ta True
te True
kn True
ml True
bn True
as True
mr True
gu True
pa True
en True


set -euo pipefail
nl -ba maya_transcribe/verify.py | sed -n '1,260p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	from dataclasses import dataclass
     4	from typing import Any
     5
     6	from maya_transcribe.lid import LIDResult
     7
     8
     9	@dataclass(frozen=True)
    10	class Verification:
    11	    decision: str  # "accept" | "human" | "reject"
    12	    reasons: list[str]
    13	    scores: dict[str, Any]
    14
    15
    16	def lid_confidence(lid: LIDResult | None) -> dict[str, Any]:
    17	    if not lid or not lid.predictions:
    18	        return {"available": False, "top1": None, "top1_prob": None, "margin": None, "confident": False}
    19	    top1 = lid.predictions[0]
    20	    top2_prob = lid.predictions[1].prob if len(lid.predictions) > 1 else 0.0
    21	    margin = float(top1.prob) - float(top2_prob)
    22	    return {
    23	        "available": True,
    24	        "top1": top1.code,
    25	        "top1_prob": float(top1.prob),
    26	        "margin": margin,
    27	        "confident": False,
    28	    }
    29
    30
    31	def verify_segment(
    32	    *,
    33	    qc: dict[str, Any],
    34	    lid: LIDResult | None,
    35	    final_text: str,
    36	    rover: dict[str, Any],
    37	    asr_summaries: dict[str, Any],
    38	    primary_asr_key: str | None = None,
    39	    ctc: dict[str, Any] | None = None,
    40	    thresholds: dict[str, Any] | None = None,
    41	) -> Verification:
    42	    """
    43	    Conservative verification gate (precision-first).
    44	    """
    45	    thresholds = thresholds or {}
    46	    min_duration = float(thresholds.get("min_duration_sec", 0.8))
    47	    max_clipping = float(thresholds.get("max_clipping_fraction", 0.01))
    48	    max_disagreement = float(thresholds.get("max_rover_disagreement_rate", 0.03))
    49	    lid_min_prob = float(thresholds.get("lid_min_prob", 0.90))
    50	    lid_min_margin = float(thresholds.get("lid_min_margin", 0.20))
    51	    require_lid = bool(thresholds.get("require_lid", True))
    52
    53	    # Whisper-style failure heuristics (see OpenAI whisper fallback logic).
    54	    min_avg_logprob = float(thresholds.get("min_avg_logprob_mean", -1.0))
    55	    max_compression_ratio = float(thresholds.get("max_compression_ratio_mean", 2.4))
    56
    57	    # CTC fit (audio-grounded) gating.
    58	    require_ctc = bool(thresholds.get("require_ctc", False))
    59	    min_ctc_margin = float(thresholds.get("min_ctc_margin", 0.0))
    60	    min_ctc_token_coverage = float(thresholds.get("min_ctc_token_coverage", 0.85))
    61	    min_ctc_token_score_mean = float(thresholds.get("min_ctc_token_score_mean", -3.0))
    62	    fail_on_ctc_span_mismatch = bool(thresholds.get("fail_on_ctc_span_mismatch", True))
    63
    64	    reasons: list[str] = []
    65	    decision = "accept"
    66
    67	    duration = float(qc.get("duration_sec") or 0.0)
    68	    clipping = float(qc.get("clipping_fraction") or 0.0)
    69	    if duration < min_duration:
    70	        decision = "reject"
    71	        reasons.append("too_short")
    72	    if clipping > max_clipping:
    73	        decision = "human" if decision != "reject" else decision
    74	        reasons.append("clipping")
    75
    76	    lidc = lid_confidence(lid)
    77	    if lidc["available"]:
    78	        lidc["confident"] = bool(lidc["top1_prob"] >= lid_min_prob and lidc["margin"] >= lid_min_margin)
    79	        if not lidc["confident"]:
    80	            decision = "human" if decision != "reject" else decision
    81	            reasons.append("lid_uncertain")
    82	    else:
    83	        reasons.append("lid_missing")
    84	        if require_lid:
    85	            decision = "human" if decision != "reject" else decision
    86
    87	    disagreement_rate = float(rover.get("disagreement_rate") or 0.0)
    88	    if disagreement_rate > max_disagreement:
    89	        decision = "human" if decision != "reject" else decision
    90	        reasons.append("asr_disagreement")
    91
    92	    final_text = (final_text or "").strip()
    93	    if "<unk>" in final_text or "<inaudible>" in final_text:
    94	        decision = "human" if decision != "reject" else decision
    95	        reasons.append("contains_unk")
    96
    97	    # Whisper-style safety: flag if any decoder claims "no speech" with high probability.
    98	    no_speech_probs = []
    99	    for summary in asr_summaries.values():
   100	        p = summary.get("no_speech_prob_mean")
   101	        if p is None:
   102	            continue
   103	        no_speech_probs.append(float(p))
   104	    if no_speech_probs and max(no_speech_probs) > 0.8:
   105	        decision = "reject"
   106	        reasons.append("no_speech")
   107
   108	    # Whisper-style hallucination heuristics: apply to the primary ASR (or rover skeleton fallback).
   109	    asr_key_for_heuristics = primary_asr_key or str(rover.get("skeleton_key") or "")
   110	    heur = asr_summaries.get(asr_key_for_heuristics, {}) if asr_key_for_heuristics else {}
   111	    avg_lp = heur.get("avg_logprob_mean")
   112	    comp = heur.get("compression_ratio_mean")
   113	    if avg_lp is not None and float(avg_lp) < min_avg_logprob:
   114	        decision = "human" if decision != "reject" else decision
   115	        reasons.append("low_logprob")
   116	    if comp is not None and float(comp) > max_compression_ratio:
   117	        decision = "human" if decision != "reject" else decision
   118	        reasons.append("high_compression_ratio")
   119
   120	    # Audio-grounded fit check (when available).
   121	    if require_ctc and (ctc is None or ctc.get("best_loss") is None):
   122	        decision = "human" if decision != "reject" else decision
   123	        reasons.append("ctc_missing")
   124	    if ctc is not None:
   125	        margin = ctc.get("margin")
   126	        if margin is not None and float(margin) < min_ctc_margin:
   127	            decision = "human" if decision != "reject" else decision
   128	            reasons.append("ctc_ambiguous")
   129
   130	        best_key = ctc.get("best_key")
   131	        scores = ctc.get("scores")
   132	        if isinstance(best_key, str) and isinstance(scores, dict):
   133	            best = scores.get(best_key) or {}
   134	            if isinstance(best, dict):
   135	                cov = best.get("token_coverage")
   136	                if cov is not None and float(cov) < min_ctc_token_coverage:
   137	                    decision = "human" if decision != "reject" else decision
   138	                    reasons.append("ctc_low_coverage")
   139	                mean_tok = best.get("token_score_mean")
   140	                if mean_tok is not None and float(mean_tok) < min_ctc_token_score_mean:
   141	                    decision = "human" if decision != "reject" else decision
   142	                    reasons.append("ctc_low_token_score")
   143	                if fail_on_ctc_span_mismatch and bool(best.get("span_mismatch")):
   144	                    decision = "human" if decision != "reject" else decision
   145	                    reasons.append("ctc_span_mismatch")
   146
   147	    scores = {
   148	        "duration_sec": duration,
   149	        "clipping_fraction": clipping,
   150	        "rover_disagreement_rate": disagreement_rate,
   151	        "lid": lidc,
   152	        "no_speech_prob_max": max(no_speech_probs) if no_speech_probs else None,
   153	        "heuristics_asr_key": asr_key_for_heuristics or None,
   154	        "avg_logprob_mean": avg_lp,
   155	        "compression_ratio_mean": comp,
   156	        "ctc": ctc,
   157	    }
   158
   159	    return Verification(decision=decision, reasons=reasons, scores=scores)


**Evaluating verification processes**

I'm looking into the verification gate settings, which default to requiring a lid and a strict probability threshold. It seems this could lead to marking many segments as human. However, I wonder if the final hypothesis might still show a transcript even if the decision is marked as human. It seems they may not alter the text in the pipeline, but I’ll need to check how decisions are recorded during this process. Let’s look into where the final selection happens!


set -euo pipefail
nl -ba maya_transcribe/pipeline.py | sed -n '1,340p'


Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 340
Output:
     1	from __future__ import annotations
     2
     3	import json
     4	import time
     5	from dataclasses import asdict
     6	from pathlib import Path
     7	from typing import Any
     8
     9	from rich.progress import track
    10
    11	from maya_transcribe.asr import ASRResult, FasterWhisperASR, TransformersWhisperASR
    12	from maya_transcribe.audio import compute_qc, ffprobe, normalize_audio
    13	from maya_transcribe.ids import segment_identity
    14	from maya_transcribe.lid import LIDResult, MMSLID
    15	from maya_transcribe.rover import rover_combine
    16	from maya_transcribe.textnorm import normalize_text
    17	from maya_transcribe.verify import verify_segment
    18
    19
    20	def _iter_audio_files(segments_dir: Path) -> list[Path]:
    21	    audio_files: list[Path] = []
    22	    for ext in ("*.wav", "*.mp3", "*.m4a", "*.flac", "*.ogg", "*.opus"):
    23	        audio_files.extend(sorted(segments_dir.rglob(ext)))
    24	    return sorted(set(audio_files))
    25
    26
    27	def _asr_summary(result: ASRResult) -> dict[str, Any]:
    28	    avg_logprobs = [s.avg_logprob for s in result.segments if s.avg_logprob is not None]
    29	    no_speech_probs = [s.no_speech_prob for s in result.segments if s.no_speech_prob is not None]
    30	    compression_ratios = [s.compression_ratio for s in result.segments if s.compression_ratio is not None]
    31
    32	    def mean(xs: list[float]) -> float | None:
    33	        return (sum(xs) / len(xs)) if xs else None
    34
    35	    return {
    36	        "language": result.language,
    37	        "language_probability": result.language_probability,
    38	        "avg_logprob_mean": mean([float(x) for x in avg_logprobs]),
    39	        "no_speech_prob_mean": mean([float(x) for x in no_speech_probs]),
    40	        "compression_ratio_mean": mean([float(x) for x in compression_ratios]),
    41	    }
    42
    43
    44	def run_pipeline(
    45	    *,
    46	    segments_dir: Path,
    47	    out_dir: Path,
    48	    lid_model_id: str = "facebook/mms-lid-256",
    49	    whisper_model: str = "large-v3",
    50	    extra_transformers_whisper_models: list[str] | None = None,
    51	    gemini_model: str | None = None,
    52	    gemini_prompt_path: Path | None = None,
    53	    gemini_max_output_tokens: int = 512,
    54	    gemini_hard_only: bool = True,
    55	    ctc_model: str | None = None,
    56	    ctc_model_map: dict[str, str] | None = None,
    57	    require_lid_backend: bool = True,
    58	    trim_silence: bool = True,
    59	    silence_threshold_db: float = -40.0,
    60	    min_silence_sec: float = 0.15,
    61	    limit: int | None = None,
    62	    thresholds: dict[str, Any] | None = None,
    63	) -> Path:
    64	    """
    65	    End-to-end run: preprocess -> LID -> ASR ensemble -> normalize -> ROVER -> verify.
    66	    Writes `out_dir/results.jsonl` and returns that path.
    67	    """
    68	    thresholds = thresholds or {}
    69	    if (ctc_model or ctc_model_map) and "require_ctc" not in thresholds:
    70	        thresholds["require_ctc"] = True
    71	    if (ctc_model or ctc_model_map) and "min_ctc_margin" not in thresholds:
    72	        thresholds["min_ctc_margin"] = 0.05
    73	    out_dir.mkdir(parents=True, exist_ok=True)
    74
    75	    audio_files = _iter_audio_files(segments_dir)
    76	    if not audio_files:
    77	        raise RuntimeError(f"No audio files found under: {segments_dir}")
    78	    if limit is not None:
    79	        audio_files = audio_files[: max(0, int(limit))]
    80
    81	    # Instantiate models once.
    82	    lid_backend: MMSLID | None
    83	    try:
    84	        lid_backend = MMSLID(model_id=lid_model_id)
    85	    except Exception as e:
    86	        if require_lid_backend:
    87	            raise RuntimeError(
    88	                "Failed to initialize MMS-LID. Install torch/transformers deps (pip install -e '.[torch]') "
    89	                "or run with require_lid_backend=False."
    90	            ) from e
    91	        lid_backend = None
    92
    93	    whisper_backend = FasterWhisperASR(model_size_or_path=whisper_model)
    94
    95	    extra_transformers_whisper_models = extra_transformers_whisper_models or []
    96	    transformers_backends: list[TransformersWhisperASR] = []
    97	    for model_id in extra_transformers_whisper_models:
    98	        transformers_backends.append(TransformersWhisperASR(model_id=model_id))
    99
   100	    ctc_scorer = None
   101	    rank_by_ctc = None
   102	    if ctc_model or ctc_model_map:
   103	        from maya_transcribe.ctc_fit import CTCFitScorer, rank_candidates_by_ctc
   104
   105	        ctc_scorer = CTCFitScorer(model_map=ctc_model_map, default_model=ctc_model)
   106	        rank_by_ctc = rank_candidates_by_ctc
   107
   108	    gemini_backend = None
   109	    if gemini_model:
   110	        try:
   111	            from maya_transcribe.gemini_asr import GeminiASR, Ge…7335 chars truncated…r_summary(v) for k, v in asr_results.items()}
   246	                verification = verify_segment(
   247	                    qc=asdict(qc),
   248	                    lid=lid,
   249	                    final_text=final_text,
   250	                    rover={
   251	                        "text": rover.text,
   252	                        "disagreement_rate": rover.disagreement_rate,
   253	                        "skeleton_key": rover.skeleton_key,
   254	                    },
   255	                    asr_summaries=asr_summaries,
   256	                    primary_asr_key=primary_asr_key,
   257	                    ctc=ctc_info,
   258	                    thresholds=thresholds,
   259	                )
   260	                t_ver1 = time.perf_counter()
   261
   262	                record = {
   263	                    "schema_version": 1,
   264	                    "segment_id": segment_id,
   265	                    "base_id": ident.base_id,
   266	                    "relpath": ident.relpath,
   267	                    "uid": ident.uid,
   268	                    "src_path": str(src),
   269	                    "wav_path": str(wav_path),
   270	                    "src_ffprobe": asdict(src_info),
   271	                    "ffprobe": asdict(info),
   272	                    "qc": asdict(qc),
   273	                    "lid": (
   274	                        None
   275	                        if lid is None
   276	                        else {
   277	                            "model": lid.model,
   278	                            "predictions": [{"code": p.code, "prob": p.prob} for p in lid.predictions],
   279	                        }
   280	                    ),
   281	                    "route": {"language_hint": route_lang},
   282	                    "asr": {
   283	                        k: {
   284	                            "model": v.model,
   285	                            "text": v.text,
   286	                            "language": v.language,
   287	                            "language_probability": v.language_probability,
   288	                            "segments": [asdict(s) for s in v.segments],
   289	                            "meta": v.meta,
   290	                        }
   291	                        for k, v in asr_results.items()
   292	                    },
   293	                    "asr_summary": asr_summaries,
   294	                    "ensemble": {
   295	                        "unique_models": ensemble_models,
   296	                        "unique_backends": ensemble_backends,
   297	                        "warnings": ensemble_warnings,
   298	                    },
   299	                    "normalized": norm_hyps,
   300	                    "rover": {
   301	                        "text": rover.text,
   302	                        "skeleton_key": rover.skeleton_key,
   303	                        "disagreement_rate": rover.disagreement_rate,
   304	                        "disagreements": [
   305	                            {"kind": d.kind, "index": d.index, "chosen": d.chosen, "options": d.options}
   306	                            for d in rover.disagreements
   307	                        ],
   308	                    },
   309	                    "ctc": ctc_info,
   310	                    "final": {"text": final_text, "source": final_source},
   311	                    "verification": {
   312	                        "decision": verification.decision,
   313	                        "reasons": verification.reasons,
   314	                        "scores": verification.scores,
   315	                    },
   316	                    "config": {
   317	                        "trim_silence": trim_silence,
   318	                        "silence_threshold_db": silence_threshold_db,
   319	                        "min_silence_sec": min_silence_sec,
   320	                        "gemini_model": gemini_model,
   321	                        "gemini_hard_only": gemini_hard_only,
   322	                        "ctc_model": ctc_model,
   323	                    },
   324	                    "timing": {
   325	                        "preprocess_sec": t_pre1 - t_pre0,
   326	                        "lid_sec": t_lid1 - t_lid0,
   327	                        "asr_sec": t_asr1 - t_asr0,
   328	                        "normalize_sec": t_norm1 - t_norm0,
   329	                        "rover_sec": t_rover1 - t_rover0,
   330	                        "ctc_sec": t_ctc1 - t_ctc0,
   331	                        "verify_sec": t_ver1 - t_ver0,
   332	                        "total_sec": time.perf_counter() - t0,
   333	                    },
   334	                }
   335
   336	                f.write(json.dumps(record, ensure_ascii=False) + "\n")
   337	                decision = verification.decision
   338	                counts[decision] = counts.get(decision, 0) + 1
   339	                timings_total.append(record["timing"]["total_sec"])
   340


**Reviewing pipeline segments**

I need to check the pipeline file because it seems truncated in the middle. I want to see the earlier part where the ASR (automatic speech recognition) results were created. It looks like I should focus on the section starting around line 108. I’ll specifically look at the segment between lines 100 and 260, just to get a complete picture of the relevant information. Let's dive in and see what we can uncover!


set -euo pipefail
nl -ba maya_transcribe/pipeline.py | sed -n '100,280p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   100	    ctc_scorer = None
   101	    rank_by_ctc = None
   102	    if ctc_model or ctc_model_map:
   103	        from maya_transcribe.ctc_fit import CTCFitScorer, rank_candidates_by_ctc
   104
   105	        ctc_scorer = CTCFitScorer(model_map=ctc_model_map, default_model=ctc_model)
   106	        rank_by_ctc = rank_candidates_by_ctc
   107
   108	    gemini_backend = None
   109	    if gemini_model:
   110	        try:
   111	            from maya_transcribe.gemini_asr import GeminiASR, GeminiConfig, load_prompt_template
   112
   113	            prompt = load_prompt_template(gemini_prompt_path or Path("Prompt.md"))
   114	            gemini_backend = GeminiASR(
   115	                config=GeminiConfig(model=gemini_model, max_output_tokens=gemini_max_output_tokens),
   116	                prompt_template=prompt,
   117	            )
   118	        except Exception as e:
   119	            raise RuntimeError("Failed to initialize Gemini backend (check GEMINI_KEY and google-genai).") from e
   120
   121	    results_path = out_dir / "results.jsonl"
   122	    summary_path = out_dir / "summary.json"
   123
   124	    counts = {"accept": 0, "human": 0, "reject": 0, "error": 0}
   125	    timings_total: list[float] = []
   126
   127	    with results_path.open("w", encoding="utf-8") as f:
   128	        for src in track(audio_files, description="Transcribing"):
   129	            ident = segment_identity(src, root=segments_dir)
   130	            segment_id = ident.segment_id
   131	            t0 = time.perf_counter()
   132
   133	            try:
   134	                # 1) Preprocess
   135	                t_pre0 = time.perf_counter()
   136	                src_info = ffprobe(src)
   137	                wav_path = out_dir / "audio" / f"{segment_id}.wav"
   138	                normalize_audio(
   139	                    src,
   140	                    wav_path,
   141	                    trim_silence=trim_silence,
   142	                    silence_threshold_db=silence_threshold_db,
   143	                    min_silence_sec=min_silence_sec,
   144	                )
   145	                info = ffprobe(wav_path)
   146	                qc = compute_qc(wav_path)
   147	                t_pre1 = time.perf_counter()
   148
   149	                # 2) LID
   150	                t_lid0 = time.perf_counter()
   151	                lid: LIDResult | None = None
   152	                if lid_backend is not None:
   153	                    lid = lid_backend.predict(wav_path, top_k=5)
   154	                t_lid1 = time.perf_counter()
   155
   156	                # Route language hint (only if confident).
   157	                route_lang = None
   158	                if lid and lid.predictions:
   159	                    top1 = lid.predictions[0]
   160	                    top2p = lid.predictions[1].prob if len(lid.predictions) > 1 else 0.0
   161	                    if float(top1.prob) >= float(thresholds.get("lid_min_prob", 0.90)) and (
   162	                        float(top1.prob) - float(top2p)
   163	                    ) >= float(thresholds.get("lid_min_margin", 0.20)):
   164	                        route_lang = top1.code
   165
   166	                # 3) ASR ensemble
   167	                t_asr0 = time.perf_counter()
   168	                asr_results: dict[str, ASRResult] = {}
   169
   170	                # Whisper: deterministic + a small sampling diversity pass.
   171	                det = whisper_backend.transcribe(wav_path, language=route_lang, beam_size=5, best_of=5, temperature=0.0)
   172	                primary_asr_key = f"{det.model}|det"
   173	                asr_results[primary_asr_key] = det
   174	                samp = whisper_backend.transcribe(
   175	                    wav_path, language=route_lang, beam_size=5, best_of=10, temperature=0.2
   176	                )
   177	                asr_results[f"{samp.model}|samp_t0.2"] = samp
   178
   179	                for tf_backend in transformers_backends:
   180	                    res = tf_backend.transcribe(wav_path, language=route_lang, num_beams=5, max_new_tokens=256)
   181	                    asr_results[res.model] = res
   182
   183	                # Gemini: optional extra hypothesis (prefer running only on "hard" segments).
   184	                if gemini_backend is not None:
   185	                    run_gemini = True
   186	                    if gemini_hard_only:
   187	                        det_sum = _asr_summary(det)
   188	                        avg_lp = det_sum.get("avg_logprob_mean")
   189	                        comp = det_sum.get("compression_ratio_mean")
   190	                        no_speech = det_sum.get("no_speech_prob_mean")
   191	                        run_gemini = bool(
   192	                            route_lang is None
   193	                            or (no_speech is not None and float(no_speech) > 0.5)
   194	                            or (avg_lp is not None and float(avg_lp) < float(thresholds.get("min_avg_logprob_mean", -1.0)))
   195	                            or (
   196	                                comp is not None
   197	                                and float(comp) > float(thresholds.get("max_compression_ratio_mean", 2.4))
   198	                            )
   199	                        )
   200
   201	                    if run_gemini:
   202	                        try:
   203	                            g = gemini_backend.transcribe(wav_path, language=route_lang)
   204	                            asr_results[g.model] = g
   205	                        except Exception:
   206	                            pass
   207
   208	                t_asr1 = time.perf_counter()
   209
   210	                ensemble_models = sorted({v.model for v in asr_results.values()})
   211	                ensemble_backends = sorted({v.model.split(":", 1)[0] for v in asr_results.values()})
   212	                ensemble_warnings: list[str] = []
   213	                if len(ensemble_models) < 2:
   214	                    ensemble_warnings.append("low_model_diversity")
   215
   216	                # 4) Normalize hypotheses
   217	                t_norm0 = time.perf_counter()
   218	                norm_hyps: dict[str, str] = {}
   219	                for k, res in asr_results.items():
   220	                    lang_hint = res.language or route_lang
   221	                    norm_hyps[k] = normalize_text(res.text, lang=lang_hint)
   222	                t_norm1 = time.perf_counter()
   223
   224	                # 5) ROVER
   225	                t_rover0 = time.perf_counter()
   226	                rover = rover_combine(norm_hyps)
   227	                t_rover1 = time.perf_counter()
   228
   229	                # 5.5) Audio-grounded fit scoring (optional): choose best candidate by CTC loss.
   230	                t_ctc0 = time.perf_counter()
   231	                ctc_info = None
   232	                final_text = rover.text
   233	                final_source = "rover"
   234	                if ctc_scorer is not None and rank_by_ctc is not None:
   235	                    candidates = {"rover": rover.text, **norm_hyps}
   236	                    ctc_info = rank_by_ctc(ctc_scorer, wav_path, candidates=candidates, lang=route_lang)
   237	                    best_key = ctc_info.get("best_key") if isinstance(ctc_info, dict) else None
   238	                    if best_key and best_key in candidates:
   239	                        final_text = candidates[best_key]
   240	                        final_source = "rover" if best_key == "rover" else f"asr:{best_key}"
   241	                t_ctc1 = time.perf_counter()
   242
   243	                # 6) Verify
   244	                t_ver0 = time.perf_counter()
   245	                asr_summaries = {k: _asr_summary(v) for k, v in asr_results.items()}
   246	                verification = verify_segment(
   247	                    qc=asdict(qc),
   248	                    lid=lid,
   249	                    final_text=final_text,
   250	                    rover={
   251	                        "text": rover.text,
   252	                        "disagreement_rate": rover.disagreement_rate,
   253	                        "skeleton_key": rover.skeleton_key,
   254	                    },
   255	                    asr_summaries=asr_summaries,
   256	                    primary_asr_key=primary_asr_key,
   257	                    ctc=ctc_info,
   258	                    thresholds=thresholds,
   259	                )
   260	                t_ver1 = time.perf_counter()
   261
   262	                record = {
   263	                    "schema_version": 1,
   264	                    "segment_id": segment_id,
   265	                    "base_id": ident.base_id,
   266	                    "relpath": ident.relpath,
   267	                    "uid": ident.uid,
   268	                    "src_path": str(src),
   269	                    "wav_path": str(wav_path),
   270	                    "src_ffprobe": asdict(src_info),
   271	                    "ffprobe": asdict(info),
   272	                    "qc": asdict(qc),
   273	                    "lid": (
   274	                        None
   275	                        if lid is None
   276	                        else {
   277	                            "model": lid.model,
   278	                            "predictions": [{"code": p.code, "prob": p.prob} for p in lid.predictions],
   279	                        }
   280	                    ),


**Evaluating ROVER pipeline**

I’m reviewing how the pipeline uses `normalize_text` on ASR outputs to create normalized hypotheses for ROVER. It seems they're intending to provide final transcripts in a normalized form, which could impact word error rates, but not dramatically. I'm noting that for non-English languages, this also removes punctuation, which could be a concern for Indic scripts. I should consider that the final output being normalized means it might not line up with the reference if that includes punctuation or digits. However, evaluations normalize the reference too, so alignment seems fair.


**Analyzing text normalization**

So, `normalize_text` removes punctuation but retains diacritics, which is interesting. For evaluations, they apply `normalize_text` to both reference and hypotheses, so that should keep comparisons consistent. However, a high word error rate likely stems from the ASR output being incorrect or possibly empty. I think it could be due to `rover_combine` returning empty if no hypotheses were generated. It might relate to the `faster-whisper` model not being installed. There are also specifics about language codes that might be causing issues during transcription. I need to clarify this further.


**Clarifying language code mismatches**

I’m noticing that the pipeline passes `route_lang` based on the top prediction from MMS-LID, which seems to be using ISO-639-3 codes like 'hin' or 'tel'. The problem arises because `faster-whisper` expects ISO-639-1 codes, or None, leading to that critical mismatch. If MMS-LID outputs ISO-3 codes, passing something like 'hin' to `faster-whisper` would definitely cause an error since that’s not valid. I need to confirm if MMS-LID models indeed output ISO-3 codes or if they might be in ISO-1. This seems essential for resolving language code errors!


**Inspecting transcription implementation**

I’m going to look closely at the `FasterWhisperASR.transcribe` implementation in `asr.py`. I suspect it might call the `to_whisper_lang` function, which seems vital for handling language codes correctly. It’s important to ensure that everything lines up properly with the expectations of the `faster-whisper` model. I'll review it to see how it handles these details and whether it offers any solutions for the language code mismatch issues I've been considering. Let's get into the code!


set -euo pipefail
nl -ba maya_transcribe/asr.py | sed -n '1,220p'