Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	import json
     4	import time
     5	from dataclasses import asdict
     6	from pathlib import Path
     7	from typing import Any
     8
     9	from rich.progress import track
    10
    11	from maya_transcribe.asr import ASRResult, FasterWhisperASR, TransformersWhisperASR
    12	from maya_transcribe.audio import compute_qc, ffprobe, normalize_audio
    13	from maya_transcribe.ids import segment_identity
    14	from maya_transcribe.lid import LIDResult, MMSLID
    15	from maya_transcribe.rover import rover_combine
    16	from maya_transcribe.textnorm import normalize_text
    17	from maya_transcribe.verify import verify_segment
    18
    19
    20	def _iter_audio_files(segments_dir: Path) -> list[Path]:
    21	    audio_files: list[Path] = []
    22	    for ext in ("*.wav", "*.mp3", "*.m4a", "*.flac", "*.ogg", "*.opus"):
    23	        audio_files.extend(sorted(segments_dir.rglob(ext)))
    24	    return sorted(set(audio_files))
    25
    26
    27	def _asr_summary(result: ASRResult) -> dict[str, Any]:
    28	    avg_logprobs = [s.avg_logprob for s in result.segments if s.avg_logprob is not None]
    29	    no_speech_probs = [s.no_speech_prob for s in result.segments if s.no_speech_prob is not None]
    30	    compression_ratios = [s.compression_ratio for s in result.segments if s.compression_ratio is not None]
    31
    32	    def mean(xs: list[float]) -> float | None:
    33	        return (sum(xs) / len(xs)) if xs else None
    34
    35	    return {
    36	        "language": result.language,
    37	        "language_probability": result.language_probability,
    38	        "avg_logprob_mean": mean([float(x) for x in avg_logprobs]),
    39	        "no_speech_prob_mean": mean([float(x) for x in no_speech_probs]),
    40	        "compression_ratio_mean": mean([float(x) for x in compression_ratios]),
    41	    }
    42
    43
    44	def run_pipeline(
    45	    *,
    46	    segments_dir: Path,
    47	    out_dir: Path,
    48	    lid_model_id: str = "facebook/mms-lid-256",
    49	    whisper_model: str = "large-v3",
    50	    extra_transformers_whisper_models: list[str] | None = None,
    51	    gemini_model: str | None = None,
    52	    gemini_prompt_path: Path | None = None,
    53	    gemini_max_output_tokens: int = 512,
    54	    gemini_hard_only: bool = True,
    55	    ctc_model: str | None = None,
    56	    ctc_model_map: dict[str, str] | None = None,
    57	    require_lid_backend: bool = True,
    58	    trim_silence: bool = True,
    59	    silence_threshold_db: float = -40.0,
    60	    min_silence_sec: float = 0.15,
    61	    limit: int | None = None,
    62	    thresholds: dict[str, Any] | None = None,
    63	) -> Path:
    64	    """
    65	    End-to-end run: preprocess -> LID -> ASR ensemble -> normalize -> ROVER -> verify.
    66	    Writes `out_dir/results.jsonl` and returns that path.
    67	    """
    68	    thresholds = thresholds or {}
    69	    if (ctc_model or ctc_model_map) and "require_ctc" not in thresholds:
    70	        thresholds["require_ctc"] = True
    71	    if (ctc_model or ctc_model_map) and "min_ctc_margin" not in thresholds:
    72	        thresholds["min_ctc_margin"] = 0.05
    73	    out_dir.mkdir(parents=True, exist_ok=True)
    74
    75	    audio_files = _iter_audio_files(segments_dir)
    76	    if not audio_files:
    77	        raise RuntimeError(f"No audio files found under: {segments_dir}")
    78	    if limit is not None:
    79	        audio_files = audio_files[: max(0, int(limit))]
    80
    81	    # Instantiate models once.
    82	    lid_backend: MMSLID | None
    83	    try:
    84	        lid_backend = MMSLID(model_id=lid_model_id)
    85	    except Exception as e:
    86	        if require_lid_backend:
    87	            raise RuntimeError(
    88	                "Failed to initialize MMS-LID. Install torch/transformers deps (pip install -e '.[torch]') "
    89	                "or run with require_lid_backend=False."
    90	            ) from e
    91	        lid_backend = None
    92
    93	    whisper_backend = FasterWhisperASR(model_size_or_path=whisper_model)
    94
    95	    extra_transformers_whisper_models = extra_transformers_whisper_models or []
    96	    transformers_backends: list[TransformersWhisperASR] = []
    97	    for model_id in extra_transformers_whisper_models:
    98	        transformers_backends.append(TransformersWhisperASR(model_id=model_id))
    99
   100	    ctc_scorer = None
   101	    rank_by_ctc = None
   102	    if ctc_model or ctc_model_map:
   103	        from maya_transcribe.ctc_fit import CTCFitScorer, rank_candidates_by_ctc
   104
   105	        ctc_scorer = CTCFitScorer(model_map=ctc_model_map, default_model=ctc_model)
   106	        rank_by_ctc = rank_candidates_by_ctc
   107
   108	    gemini_backend = None
   109	    if gemini_model:
   110	        try:
   111	            from maya_transcribe.gemini_asr import GeminiASR, GeminiConfig, load_prompt_template
   112
   113	            prompt = load_prompt_template(gemini_prompt_path or Path("Prompt.md"))
   114	            gemini_backend = GeminiASR(
   115	                config=GeminiConfig(model=gemini_model, max_output_tokens=gemini_max_output_tokens),
   116	                prompt_template=prompt,
   117	            )
   118	        except Exception as e:
   119	            raise RuntimeError("Failed to initialize Gemini backend (check GEMINI_KEY and google-genai).") from e
   120
   121	    results_path = out_dir / "results.jsonl"
   122	    summary_path = out_dir / "summary.json"
   123
   124	    counts = {"accept": 0, "human": 0, "reject": 0, "error": 0}
   125	    timings_total: list[float] = []
   126
   127	    seen: dict[str, int] = {}
   128	    with results_path.open("w", encoding="utf-8") as f:
   129	        for src in track(audio_files, description="Transcribing"):
   130	            ident = segment_identity(src, root=segments_dir, seen=seen)
   131	            segment_id = ident.segment_id
   132	            t0 = time.perf_counter()
   133
   134	            try:
   135	                # 1) Preprocess
   136	                t_pre0 = time.perf_counter()
   137	                src_info = ffprobe(src)
   138	                wav_path = out_dir / "audio" / f"{segment_id}.wav"
   139	                normalize_audio(
   140	                    src,
   141	                    wav_path,
   142	                    trim_silence=trim_silence,
   143	                    silence_threshold_db=silence_threshold_db,
   144	                    min_silence_sec=min_silence_sec,
   145	                )
   146	                info = ffprobe(wav_path)
   147	                qc = compute_qc(wav_path)
   148	                t_pre1 = time.perf_counter()
   149
   150	                # 2) LID
   151	                t_lid0 = time.perf_counter()
   152	                lid: LIDResult | None = None
   153	                if lid_backend is not None:
   154	                    lid = lid_backend.predict(wav_path, top_k=5)
   155	                t_lid1 = time.perf_counter()
   156
   157	                # Route language hint (only if confident).
   158	                route_lang = None
   159	                if lid and lid.predictions:
   160	                    top1 = lid.predictions[0]
   161	                    top2p = lid.predictions[1].prob if len(lid.predictions) > 1 else 0.0
   162	                    if float(top1.prob) >= float(thresholds.get("lid_min_prob", 0.90)) and (
   163	                        float(top1.prob) - float(top2p)
   164	                    ) >= float(thresholds.get("lid_min_margin", 0.20)):
   165	                        route_lang = top1.code
   166
   167	                # 3) ASR ensemble
   168	                t_asr0 = time.perf_counter()
   169	                asr_results: dict[str, ASRResult] = {}
   170


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Second-round review (based on current repo state)**

**What’s solid**
- Prompt is now correctly anti-hallucination + single-output (`Prompt.md:6`), and Gemini output parsing is strict (`maya_transcribe/gemini_asr.py:78`).
- Gemini is integrated as an *optional* hypothesis with retries/backoff and raw-output stored for audit (`maya_transcribe/gemini_asr.py:156`).
- Pipeline now has optional CTC fit scoring + final transcript selection (`maya_transcribe/pipeline.py:224`) and verification uses LID + disagreement + Whisper heuristics + CTC ambiguity (`maya_transcribe/verify.py:31`).
- Eval now reports WER + CER + duration buckets + tail stats, defaulting to `final` (`maya_transcribe/eval.py:46`, `maya_transcribe/cli.py:267`).

**Key robustness issues / edge cases to fix next**
- **Segment IDs aren’t stable across partial runs.** `segment_identity()` uses `base_id` for the first occurrence and appends `-uid` only on collisions (`maya_transcribe/ids.py:16`). This means `--limit` / different traversal orders can change IDs, breaking reference alignment and reproducibility. For a quality pipeline, make `segment_id` *always* include the hash (or use `uid` as the ID) and keep `base_id` only as metadata.
- **Gemini language hint is currently an ISO-ish code (e.g., `hin`, `tam`) or `"unknown"`.** That’s not a great instruction for “native script” behavior (`maya_transcribe/gemini_asr.py:145`). Either:
  - map codes → human-readable names for the prompt (Hindi/Telugu/…), and/or
  - update the prompt to explicitly handle `unknown` (“detect and use the correct native script”), otherwise Gemini may output inconsistent scripts and can pollute ROVER if CTC is off.
- **`parse_strict_output()` is likely too brittle for production.** It rejects any punctuation and any multi-line transcript (`maya_transcribe/gemini_asr.py:90`). In practice, models sometimes insert line breaks/punctuation even with strict prompts. Consider “salvage mode”: join extra lines into one transcript and strip punctuation (since you normalize later anyway) rather than dropping the hypothesis entirely.
- **Ensemble diversity is still optional.** Default run is Whisper det + Whisper sample only (`maya_transcribe/pipeline.py:171`), which won’t minimize WER on Indic. Make sure you’re always passing at least one Indic-specialist checkpoint via `--transformers-whisper-model` (IndicWhisper), and ideally a non-Whisper-family model too.
- **CTC fit scoring needs guardrails for tokenization/OOV.** Current scoring uses `tokenizer(text_norm)` blindly (`maya_transcribe/ctc_fit.py:79`). Add checks like “% of tokens are unk” / “contains characters not in vocab” and treat that candidate as unscorable, otherwise losses can be misleading across scripts.
- **Verification still lacks the strongest audio-grounded signal: alignment coverage.** CTC loss margin is helpful (`maya_transcribe/verify.py:117`), but you’ll want a coverage/fit metric that can catch “mostly wrong but fluent” outputs (e.g., CTC forced alignment coverage, or word-level alignment confidence), especially on YouTube artifacts (cross-talk, residual music, breathy starts).

**Evaluation upgrades to really stress-test WER minimization**
- Add **oracle analysis**: compute WER/CER for every candidate (`asr[*].text`, `rover.text`, `final.text`) and report:
  - `oracle_best_wer` (minimum achievable if you always picked the best hypothesis),
  - `selection_gap = final_wer - oracle_best_wer`.
  This tells you whether to add better models (oracle high) vs improve selection/gating (gap high).
- Add **acceptance tradeoff curves**: vary thresholds (disagreement, logprob, CTC margin) and report `(accept_rate, WER_accept_only, CER_accept_only)`. This is the right way to tune “precision-first” acceptance.
- Build a “challenge set” separate from the gold set: code-switch, fast speech, numbers, proper nouns, short 1–3s clips, mild noise/music bleed.

**On “superior model” for languages you can’t validate (Gemini-3-Pro, etc.)**
- Treat Gemini-3-Pro (or any stronger audio LLM) as **a fallback hypothesis only on hard segments**, never as ground truth. Your current `gemini_hard_only` mechanism is the right place to plug it in (`maya_transcribe/pipeline.py:184`).
- A good escalation policy: run `gemini-2.0-flash` first; if parsing fails or CTC is ambiguous, escalate to `gemini-3-pro-preview`. Still require the same CTC/verification gate before acceptance.

If you want, paste your agent’s exact CLI “Try It” commands again; one of them looks off (the `wav2vec2-base-960h --gemini-model …` part doesn’t match the current `maya-transcribe gemini` flags in `maya_transcribe/cli.py:188`).


# Context from my IDE setup:

## Open tabs:
- .env: .env
- Prompt.md: Prompt.md
- master.md: master.md
- Agents.md: Agents.md
- task.md: task.md

## My request for Codex:
Review Fixes Landed

Stable, reproducible segment_id (always base_id-uid, 64-bit hash): ids.py (line 16) (this will change IDs vs older runs, so older refs.jsonl won’t match).
Gemini language hint now uses human-friendly names/scripts for prompting (and “Unknown → auto-detect”): language.py (line 80), gemini_asr.py (line 153).
Gemini parsing is less brittle (salvage multi-line, strip punctuation, still records parse flags): gemini_asr.py (line 100).
CTC scoring guardrails + alignment-coverage metrics (unk-ratio filter, token coverage/score stats): ctc_fit.py (line 64).
Verification gate now uses CTC coverage metrics in addition to margin (flags ctc_low_coverage / ctc_low_token_score / ctc_span_mismatch): verify.py (line 31).
Eval now includes oracle stats (oracle-best WER/CER + selection gap): eval.py (line 50).
Metrics To Surface In UI (already in results.jsonl)

Primary decisioning: verification.decision, verification.reasons, verification.scores (pipeline.py (line 263)).
Audio/QC: src_ffprobe.duration_sec vs ffprobe.duration_sec (trim impact), qc.rms_dbfs, qc.peak_dbfs, qc.clipping_fraction.
LID: lid.predictions[] and verification.scores.lid.
Ensemble health: ensemble.unique_models, ensemble.warnings (low_model_diversity): pipeline.py (line 293).
Hypotheses + comparisons: asr[*].text, asr_summary, rover.text + rover.disagreements, ctc.scores, and final.text + final.source.

Addressed. can you now search the internet and find benchmarks on few datasets that we can actually use for in this scenario here and get some accuracy scores or WER scores on our pipeline ? how about datasets from ai4bharath any benchmarking datasets ? can you search and give ideas to my other agent ? and also check if the code is robust enough now for us to do a benchmark on few samples from various languages and analyse patterns between languages so we can tune accordingly.

1. Firstly, see if all code is robust and we are good to perform benchmarks on opensource data
2. give ideas and how we can perform these benchmarks, so we can get language-wise performance or statistics
3. handover script/instructions(inchat, dont create new files) to my other agent to do these benchmarks and analyse patterns to better conclude this before we move to a UI based diff analysis and see if human intervention is required at any scale or not.


Summary: auto


**Evaluating benchmarking plans**

The user didn't specifically ask to run tests, but they want me to evaluate robustness and propose a benchmarking plan. I can do a quick local test with existing sample data, but they prefer open-source datasets across languages. With network access, using repositories like Hugging Face or OpenSLR is possible, although they might be heavy. I need to ensure our code is robust, verifying CLI commands, and evaluating scripts and IDs. We should avoid destructive actions while proceeding.


**Considering dataset suggestions**

I need to be careful about handling sensitive information, especially around user secrets like those in `.env` files. While I've printed a list of keys, I shouldn't expose their values. It's important to provide strong suggestions for datasets. I might include the AI4Bharat "Vistaar" dataset, "IndicVoices," and possibly "Bhashini," though the latter may require a license. I can look online for dataset names and evaluation splits, using tools like `curl`. Checking the updated code is my next step. I'll analyze the files to see recent modifications.


nl -ba maya_transcribe/ids.py | sed -n '1,140p'

nl -ba maya_transcribe/language.py | sed -n '1,220p'

nl -ba maya_transcribe/gemini_asr.py | sed -n '1,260p'

nl -ba maya_transcribe/ctc_fit.py | sed -n '1,260p'

nl -ba maya_transcribe/verify.py | sed -n '1,240p'

nl -ba maya_transcribe/eval.py | sed -n '1,260p'

nl -ba maya_transcribe/pipeline.py | sed -n '1,360p'


Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 1421
Output:
     1	from __future__ import annotations
     2
     3	import hashlib
     4	from dataclasses import dataclass
     5	from pathlib import Path
     6
     7
     8	@dataclass(frozen=True)
     9	class SegmentIdentity:
    10	    segment_id: str
    11	    base_id: str
    12	    relpath: str
    13	    uid: str
    14
    15
    16	def segment_identity(src: Path, root: Path) -> SegmentIdentity:
    17	    """
    18	    Produce a stable segment identity and prevent collisions when stems repeat.
    19	    - `base_id`: `src.stem`
    20	    - `relpath`: path relative to `root` (or absolute as fallback)
    21	    - `uid`: sha1(relpath) prefix
    22	    - `segment_id`: always includes uid for stability across runs
    23	    """
    24	    base_id = src.stem
    25	    try:
    26	        relpath = src.relative_to(root).as_posix()
    27	    except Exception:
    28	        relpath = str(src)
    29
    30	    # Use 64 bits of hash to make collisions negligibly unlikely at large scale.
    31	    uid = hashlib.sha1(relpath.encode("utf-8")).hexdigest()[:16]
    32	    segment_id = f"{base_id}-{uid}"
    33
    34	    return SegmentIdentity(segment_id=segment_id, base_id=base_id, relpath=relpath, uid=uid)
     1	from __future__ import annotations
     2
     3	ISO3_TO_WHISPER_LANG: dict[str, str] = {
     4	    "eng": "en",
     5	    "hin": "hi",
     6	    "mar": "mr",
     7	    "guj": "gu",
     8	    "pan": "pa",
     9	    "tam": "ta",
    10	    "tel": "te",
    11	    "kan": "kn",
    12	    "mal": "ml",
    13	    "ben": "bn",
    14	    "asm": "as",
    15	    "ori": "or",
    16	    "ory": "or",
    17	}
    18
    19	# Human-friendly names and scripts (for prompts/UI).
    20	LANG_CODE_TO_NAME: dict[str, str] = {
    21	    "eng": "English",
    22	    "en": "English",
    23	    "hin": "Hindi",
    24	    "hi": "Hindi",
    25	    "mar": "Marathi",
    26	    "mr": "Marathi",
    27	    "guj": "Gujarati",
    28	    "gu": "Gujarati",
    29	    "pan": "Punjabi",
    30	    "pa": "Punjabi",
    31	    "tam": "Tamil",
    32	    "ta": "Tamil",
    33	    "tel": "Telugu",
    34	    "te": "Telugu",
    35	    "kan": "Kannada",
    36	    "kn": "Kannada",
    37	    "mal": "Malayalam",
    38	    "ml": "Malayalam",
    39	    "ben": "Bengali",
    40	    "bn": "Bengali",
    41	    "asm": "Assamese",
    42	    "as": "Assamese",
    43	    "ori": "Odia",
    44	    "ory": "Odia",
    45	    "or": "Odia",
    46	}
    47
    48	LANG_NAME_TO_SCRIPT: dict[str, str] = {
    49	    "English": "Latin",
    50	    "Hindi": "Devanagari",
    51	    "Marathi": "Devanagari",
    52	    "Gujarati": "Gujarati",
    53	    "Punjabi": "Gurmukhi",
    54	    "Tamil": "Tamil",
    55	    "Telugu": "Telugu",
    56	    "Kannada": "Kannada",
    57	    "Malayalam": "Malayalam",
    58	    "Bengali": "Bengali",
    59	    "Assamese": "Assamese",
    60	    "Odia": "Odia",
    61	}
    62
    63
    64	def to_whisper_lang(lang_code: str | None) -> str | None:
    65	    if not lang_code:
    66	        return None
    67	    lang_code = lang_code.strip().lower()
    68	    if len(lang_code) == 2:
    69	        return lang_code
    70	    return ISO3_TO_WHISPER_LANG.get(lang_code, None)
    71
    72
    73	def to_language_name(lang_code: str | None) -> str | None:
    74	    if not lang_code:
    75	        return None
    76	    lang_code = lang_code.strip().lower()
    77	    return LANG_CODE_TO_NAME.get(lang_code)
    78
    79
    80	def to_prompt_language_hint(lang_code: str | None) -> str:
    81	    """
    82	    Return a human-friendly language hint string for prompting audio-native models.
    83	    """
    84	    if not lang_code:
    85	        return "Unknown (auto-detect primary spoken language)"
    86	    lang_code_norm = lang_code.strip().lower()
    87	    name = LANG_CODE_TO_NAME.get(lang_code_norm)
    88	    if not name:
    89	        return f"{lang_code_norm} (auto-detect primary spoken language)"
    90	    script = LANG_NAME_TO_SCRIPT.get(name)
    91	    if script:
    92	        return f"{name} ({script} script)"
    93	    return name
     1	from __future__ import annotations
     2
     3	import os
     4	import time
     5	import unicodedata
     6	from dataclasses import dataclass
     7	from pathlib import Path
     8	from typing import Any
     9
    10	from maya_transcribe.asr import ASRResult
    11	from maya_transcribe.language import to_prompt_language_hint
    12	from maya_transcribe.textnorm import normalize_text
    13
    14
    15	def _get_api_key() -> str:
    16	    # Support common env var names; `.env` uses GEMINI_KEY in this repo.
    17	    key = (
    18	        os.getenv("GEMINI_KEY")
    19	        or os.getenv("GEMINI_API_KEY")
    20	        or os.getenv("GOOGLE_API_KEY")
    21	        or os.getenv("GOOGLE_GENAI_API_KEY")
    22	    )
    23	    if not key:
    24	        raise RuntimeError("Missing Gemini API key (set GEMINI_KEY or GOOGLE_API_K…53783 chars truncated… segment_id,
   265	                    "base_id": ident.base_id,
   266	                    "relpath": ident.relpath,
   267	                    "uid": ident.uid,
   268	                    "src_path": str(src),
   269	                    "wav_path": str(wav_path),
   270	                    "src_ffprobe": asdict(src_info),
   271	                    "ffprobe": asdict(info),
   272	                    "qc": asdict(qc),
   273	                    "lid": (
   274	                        None
   275	                        if lid is None
   276	                        else {
   277	                            "model": lid.model,
   278	                            "predictions": [{"code": p.code, "prob": p.prob} for p in lid.predictions],
   279	                        }
   280	                    ),
   281	                    "route": {"language_hint": route_lang},
   282	                    "asr": {
   283	                        k: {
   284	                            "model": v.model,
   285	                            "text": v.text,
   286	                            "language": v.language,
   287	                            "language_probability": v.language_probability,
   288	                            "segments": [asdict(s) for s in v.segments],
   289	                            "meta": v.meta,
   290	                        }
   291	                        for k, v in asr_results.items()
   292	                    },
   293	                    "asr_summary": asr_summaries,
   294	                    "ensemble": {
   295	                        "unique_models": ensemble_models,
   296	                        "unique_backends": ensemble_backends,
   297	                        "warnings": ensemble_warnings,
   298	                    },
   299	                    "normalized": norm_hyps,
   300	                    "rover": {
   301	                        "text": rover.text,
   302	                        "skeleton_key": rover.skeleton_key,
   303	                        "disagreement_rate": rover.disagreement_rate,
   304	                        "disagreements": [
   305	                            {"kind": d.kind, "index": d.index, "chosen": d.chosen, "options": d.options}
   306	                            for d in rover.disagreements
   307	                        ],
   308	                    },
   309	                    "ctc": ctc_info,
   310	                    "final": {"text": final_text, "source": final_source},
   311	                    "verification": {
   312	                        "decision": verification.decision,
   313	                        "reasons": verification.reasons,
   314	                        "scores": verification.scores,
   315	                    },
   316	                    "config": {
   317	                        "trim_silence": trim_silence,
   318	                        "silence_threshold_db": silence_threshold_db,
   319	                        "min_silence_sec": min_silence_sec,
   320	                        "gemini_model": gemini_model,
   321	                        "gemini_hard_only": gemini_hard_only,
   322	                        "ctc_model": ctc_model,
   323	                    },
   324	                    "timing": {
   325	                        "preprocess_sec": t_pre1 - t_pre0,
   326	                        "lid_sec": t_lid1 - t_lid0,
   327	                        "asr_sec": t_asr1 - t_asr0,
   328	                        "normalize_sec": t_norm1 - t_norm0,
   329	                        "rover_sec": t_rover1 - t_rover0,
   330	                        "ctc_sec": t_ctc1 - t_ctc0,
   331	                        "verify_sec": t_ver1 - t_ver0,
   332	                        "total_sec": time.perf_counter() - t0,
   333	                    },
   334	                }
   335
   336	                f.write(json.dumps(record, ensure_ascii=False) + "\n")
   337	                decision = verification.decision
   338	                counts[decision] = counts.get(decision, 0) + 1
   339	                timings_total.append(record["timing"]["total_sec"])
   340
   341	            except Exception as e:  # pragma: no cover
   342	                counts["error"] += 1
   343	                f.write(
   344	                    json.dumps(
   345	                        {"segment_id": segment_id, "src_path": str(src), "error": str(e)}, ensure_ascii=False
   346	                    )
   347	                    + "\n"
   348	                )
   349
   350	    summary = {
   351	        "segments_total": len(audio_files),
   352	        "counts": counts,
   353	        "avg_total_sec": (sum(timings_total) / len(timings_total)) if timings_total else None,
   354	        "p50_total_sec": _percentile(timings_total, 0.50),
   355	        "p90_total_sec": _percentile(timings_total, 0.90),
   356	    }
   357	    summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
   358	    return results_path
   359
   360


nl -ba maya_transcribe/gemini_asr.py | sed -n '60,240p'

nl -ba maya_transcribe/ctc_fit.py | sed -n '1,260p'

nl -ba maya_transcribe/verify.py | sed -n '1,260p'

nl -ba maya_transcribe/eval.py | sed -n '1,340p'

nl -ba maya_transcribe/pipeline.py | sed -n '150,360p'


Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 1098
Output:
    60	        "Input: Audio file attached\n"
    61	        "Language hint (primary): <language>\n\n"
    62	        "Task:\n"
    63	        "- Transcribe only what is clearly spoken in the audio.\n"
    64	        "- Do NOT guess or “fill in” missing words. If unclear, output <unk>.\n"
    65	        "- Preserve repetitions, filler words, stutters, and false starts.\n"
    66	        "- Do NOT correct grammar, spelling, or pronunciation.\n"
    67	        "- Do NOT translate or summarize.\n"
    68	        "- Do NOT add punctuation.\n\n"
    69	        "Code-switching:\n"
    70	        "- Use the native script for the primary language.\n"
    71	        "- If English words are spoken, keep them in Latin script as spoken.\n\n"
    72	        "OUTPUT FORMAT (exact):\n"
    73	        "Output 1: Transcript\n"
    74	        "<single transcript line>\n"
    75	        "Do not output anything else.\n"
    76	    ).strip()
    77
    78	@dataclass(frozen=True)
    79	class ParsedGeminiOutput:
    80	    transcript: str
    81	    had_label: bool
    82	    joined_lines: bool
    83	    stripped_punctuation: int
    84	    extra_outputs_present: bool
    85
    86
    87	def _strip_punctuation_to_space(text: str) -> tuple[str, int]:
    88	    out: list[str] = []
    89	    removed = 0
    90	    for ch in text:
    91	        if unicodedata.category(ch).startswith("P"):
    92	            out.append(" ")
    93	            removed += 1
    94	        else:
    95	            out.append(ch)
    96	    cleaned = " ".join("".join(out).split())
    97	    return cleaned, removed
    98
    99
   100	def parse_strict_output(text: str) -> ParsedGeminiOutput:
   101	    """
   102	    Extract the transcript from the strict output format.
   103
   104	    Production note: even with strict prompts, some models may add punctuation or line breaks.
   105	    We salvage by joining lines and stripping punctuation (since the pipeline normalizes later anyway).
   106	    """
   107	    text = (text or "").replace("\r\n", "\n").replace("\r", "\n").strip()
   108	    if not text:
   109	        raise ValueError("empty_response")
   110
   111	    raw_lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
   112	    lines = [ln for ln in raw_lines if ln != "```" and not ln.startswith("```")]
   113	    if not lines:
   114	        raise ValueError("empty_response")
   115
   116	    lowered = "\n".join(lines).lower()
   117	    extra_outputs_present = ("output 2" in lowered) or ("output 3" in lowered) or ("output 4" in lowered)
   118
   119	    had_label = False
   120	    transcript_lines: list[str] = []
   121
   122	    def is_output_label(line: str) -> bool:
   123	        return line.lower().startswith("output ")
   124
   125	    label_idx = None
   126	    for i, ln in enumerate(lines):
   127	        if ln.lower().replace(" ", "") == "output1:transcript":
   128	            label_idx = i
   129	            break
   130
   131	    if label_idx is not None:
   132	        had_label = True
   133	        for ln in lines[label_idx + 1 :]:
   134	            if is_output_label(ln):
   135	                break
   136	            transcript_lines.append(ln)
   137	    else:
   138	        for ln in lines:
   139	            if is_output_label(ln):
   140	                break
   141	            transcript_lines.append(ln)
   142
   143	    if not transcript_lines:
   144	        raise ValueError("empty_transcript")
   145
   146	    joined_lines = len(transcript_lines) > 1
   147	    transcript = " ".join(transcript_lines).strip()
   148	    transcript, stripped = _strip_punctuation_to_space(transcript)
   149
   150	    if not transcript:
   151	        raise ValueError("empty_transcript")
   152
   153	    return ParsedGeminiOutput(
   154	        transcript=transcript,
   155	        had_label=had_label,
   156	        joined_lines=joined_lines,
   157	        stripped_punctuation=stripped,
   158	        extra_outputs_present=extra_outputs_present,
   159	    )
   160
   161
   162	@dataclass(frozen=True)
   163	class GeminiConfig:
   164	    model: str = "gemini-2.0-flash"
   165	    temperature: float = 0.0
   166	    max_output_tokens: int = 512
   167	    response_mime_type: str = "text/plain"
   168	    retries: int = 3
   169	    initial_backoff_sec: float = 1.0
   170
   171
   172	class GeminiASR:
   173	    def __init__(self, config: GeminiConfig | None = None, prompt_template: str | None = None) -> None:
   174	        try:
   175	            from google import genai
   176	            from google.genai import types
   177	        except Exception as e:  # pragma: no cover
   178	            raise RuntimeError("GeminiASR requires `google-genai` (pip install google-genai)") from e
   179
   180	        self._genai = genai
   181	        self._types = types
   182	        self.config = config or GeminiConfig()
   183	        self._client = genai.Client(api_key=_get_api_key())
   …40893 chars truncated… segment_id,
   265	                    "base_id": ident.base_id,
   266	                    "relpath": ident.relpath,
   267	                    "uid": ident.uid,
   268	                    "src_path": str(src),
   269	                    "wav_path": str(wav_path),
   270	                    "src_ffprobe": asdict(src_info),
   271	                    "ffprobe": asdict(info),
   272	                    "qc": asdict(qc),
   273	                    "lid": (
   274	                        None
   275	                        if lid is None
   276	                        else {
   277	                            "model": lid.model,
   278	                            "predictions": [{"code": p.code, "prob": p.prob} for p in lid.predictions],
   279	                        }
   280	                    ),
   281	                    "route": {"language_hint": route_lang},
   282	                    "asr": {
   283	                        k: {
   284	                            "model": v.model,
   285	                            "text": v.text,
   286	                            "language": v.language,
   287	                            "language_probability": v.language_probability,
   288	                            "segments": [asdict(s) for s in v.segments],
   289	                            "meta": v.meta,
   290	                        }
   291	                        for k, v in asr_results.items()
   292	                    },
   293	                    "asr_summary": asr_summaries,
   294	                    "ensemble": {
   295	                        "unique_models": ensemble_models,
   296	                        "unique_backends": ensemble_backends,
   297	                        "warnings": ensemble_warnings,
   298	                    },
   299	                    "normalized": norm_hyps,
   300	                    "rover": {
   301	                        "text": rover.text,
   302	                        "skeleton_key": rover.skeleton_key,
   303	                        "disagreement_rate": rover.disagreement_rate,
   304	                        "disagreements": [
   305	                            {"kind": d.kind, "index": d.index, "chosen": d.chosen, "options": d.options}
   306	                            for d in rover.disagreements
   307	                        ],
   308	                    },
   309	                    "ctc": ctc_info,
   310	                    "final": {"text": final_text, "source": final_source},
   311	                    "verification": {
   312	                        "decision": verification.decision,
   313	                        "reasons": verification.reasons,
   314	                        "scores": verification.scores,
   315	                    },
   316	                    "config": {
   317	                        "trim_silence": trim_silence,
   318	                        "silence_threshold_db": silence_threshold_db,
   319	                        "min_silence_sec": min_silence_sec,
   320	                        "gemini_model": gemini_model,
   321	                        "gemini_hard_only": gemini_hard_only,
   322	                        "ctc_model": ctc_model,
   323	                    },
   324	                    "timing": {
   325	                        "preprocess_sec": t_pre1 - t_pre0,
   326	                        "lid_sec": t_lid1 - t_lid0,
   327	                        "asr_sec": t_asr1 - t_asr0,
   328	                        "normalize_sec": t_norm1 - t_norm0,
   329	                        "rover_sec": t_rover1 - t_rover0,
   330	                        "ctc_sec": t_ctc1 - t_ctc0,
   331	                        "verify_sec": t_ver1 - t_ver0,
   332	                        "total_sec": time.perf_counter() - t0,
   333	                    },
   334	                }
   335
   336	                f.write(json.dumps(record, ensure_ascii=False) + "\n")
   337	                decision = verification.decision
   338	                counts[decision] = counts.get(decision, 0) + 1
   339	                timings_total.append(record["timing"]["total_sec"])
   340
   341	            except Exception as e:  # pragma: no cover
   342	                counts["error"] += 1
   343	                f.write(
   344	                    json.dumps(
   345	                        {"segment_id": segment_id, "src_path": str(src), "error": str(e)}, ensure_ascii=False
   346	                    )
   347	                    + "\n"
   348	                )
   349
   350	    summary = {
   351	        "segments_total": len(audio_files),
   352	        "counts": counts,
   353	        "avg_total_sec": (sum(timings_total) / len(timings_total)) if timings_total else None,
   354	        "p50_total_sec": _percentile(timings_total, 0.50),
   355	        "p90_total_sec": _percentile(timings_total, 0.90),
   356	    }
   357	    summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
   358	    return results_path
   359
   360


nl -ba maya_transcribe/ctc_fit.py | sed -n '1,240p'

nl -ba maya_transcribe/verify.py | sed -n '1,260p'

nl -ba maya_transcribe/eval.py | sed -n '1,340p'

nl -ba maya_transcribe/pipeline.py | sed -n '1,220p'

nl -ba maya_transcribe/pipeline.py | sed -n '220,360p'

nl -ba maya_transcribe/gemini_asr.py | sed -n '170,320p'


Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 1133
Output:
     1	from __future__ import annotations
     2
     3	from dataclasses import dataclass
     4	from pathlib import Path
     5	from typing import Any
     6
     7	from maya_transcribe.audio import decode_pcm16
     8	from maya_transcribe.textnorm import normalize_text
     9
    10
    11	@dataclass(frozen=True)
    12	class CTCScore:
    13	    model_id: str
    14	    loss: float
    15	    text: str
    16	    unk_ratio: float | None
    17	    target_len: int
    18	    token_score_mean: float | None
    19	    token_score_min: float | None
    20	    token_coverage: float | None
    21	    token_logp_threshold: float | None
    22	    span_count: int | None
    23	    span_mismatch: bool | None
    24
    25
    26	class _CTCModel:
    27	    def __init__(self, model_id: str, device: str | None = None) -> None:
    28	        try:
    29	            import torch
    30	            from transformers import AutoModelForCTC, AutoProcessor
    31	        except Exception as e:  # pragma: no cover
    32	            raise RuntimeError(
    33	                "CTC scoring requires torch + transformers. Install with: pip install -e '.[torch]'"
    34	            ) from e
    35
    36	        if device is None:
    37	            device = "cuda" if torch.cuda.is_available() else "cpu"
    38
    39	        self.model_id = model_id
    40	        self.device = device
    41	        self._torch = torch
    42	        self._processor = AutoProcessor.from_pretrained(model_id)
    43	        self._model = AutoModelForCTC.from_pretrained(model_id).to(device)
    44	        self._model.eval()
    45
    46	        tokenizer = getattr(self._processor, "tokenizer", None)
    47	        self._force_uppercase = False
    48	        if tokenizer is not None and hasattr(tokenizer, "get_vocab"):
    49	            vocab = tokenizer.get_vocab()
    50	            # Common for wav2vec2 CTC: vocab contains A-Z but not a-z; transcripts should be uppercased.
    51	            self._force_uppercase = ("A" in vocab) and ("a" not in vocab)
    52
    53	        blank = getattr(self._model.config, "pad_token_id", None)
    54	        if blank is None and getattr(self._processor, "tokenizer", None) is not None:
    55	            blank = getattr(self._processor.tokenizer, "pad_token_id", None)
    56	        self.blank_id = int(blank) if blank is not None else 0
    57
    58	    def score(
    59	        self,
    60	        audio_path: Path,
    61	        text: str,
    62	        lang: str | None = None,
    63	        *,
    64	        max_unk_ratio: float = 0.2,
    65	        token_logp_threshold: float = -3.0,
    66	    ) -> CTCScore:
    67	        import torch.nn.functional as F
    68	        import warnings
    69
    70	        if not text:
    71	            raise ValueError("empty_text")
    72	        if "<unk>" in text or "<inaudible>" in text:
    73	            raise ValueError("contains_unk")
    74
    75	        wav, sr = decode_pcm16(audio_path, sample_rate=16000, mono=True)
    76	        if wav.size == 0:
    77	            raise ValueError("empty_audio")
    78
    79	        # Conservative normalization helps tokenizer compatibility.
    80	        text_norm = normalize_text(text, lang=lang)
    81	        if not text_norm:
    82	            raise ValueError("empty_text")
    83	        if self._force_uppercase:
    84	            text_norm = text_norm.upper()
    85
    86	        inputs = self._processor(wav, sampling_rate=sr, return_tensors="pt")
    87
    88	        # Common key names: input_values (wav2vec2) or input_features (some processors).
    89	        if "input_values" in inputs:
    90	            model_inputs = {"input_values": inputs["input_values"].to(self.device)}
    91	        elif "input_features" in inputs:
    92	            model_inputs = {"input_features": inputs["input_features"].to(self.device)}
    93	        else:
    94	            # Best-effort: move all tensors to device.
    95	            model_inputs = {k: v.to(self.device) for k, v in inputs.items() if hasattr(v, "to")}
    96
    97	        with self._torch.no_grad():
    98	            logits = self._model(**model_inputs).logits  # (B,T,C)
    99	            log_probs = logits.log_softmax(dim=-1)  # (B,T,C)
   100
   101	        # Tokenize targets.
   102	        tokenizer = getattr(self._processor, "tokenizer", None)
   103	        if tokenizer is None:
   104	            raise ValueError("missing_tokenizer")
   105	        tokenized = tokenizer(text_norm, add_special_tokens=False)
   106	        target_ids = tokenized.get("input_ids")
   107	        if not target_ids:
   108	            raise ValueError("empty_targets")
   109
   110	        unk_id = getattr(tokenizer, "unk_token_id", None)
   111	        unk_ratio: float | None = None
   112	        if unk_id is not None:
   113	            unk_count = sum(1 for tid in target_ids if int(tid) == int(unk_id))
   114	            unk_ratio = fl…43289 chars truncated…ps(
   345	                        {"segment_id": segment_id, "src_path": str(src), "error": str(e)}, ensure_ascii=False
   346	                    )
   347	                    + "\n"
   348	                )
   349
   350	    summary = {
   351	        "segments_total": len(audio_files),
   352	        "counts": counts,
   353	        "avg_total_sec": (sum(timings_total) / len(timings_total)) if timings_total else None,
   354	        "p50_total_sec": _percentile(timings_total, 0.50),
   355	        "p90_total_sec": _percentile(timings_total, 0.90),
   356	    }
   357	    summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
   358	    return results_path
   359
   360
   170
   171
   172	class GeminiASR:
   173	    def __init__(self, config: GeminiConfig | None = None, prompt_template: str | None = None) -> None:
   174	        try:
   175	            from google import genai
   176	            from google.genai import types
   177	        except Exception as e:  # pragma: no cover
   178	            raise RuntimeError("GeminiASR requires `google-genai` (pip install google-genai)") from e
   179
   180	        self._genai = genai
   181	        self._types = types
   182	        self.config = config or GeminiConfig()
   183	        self._client = genai.Client(api_key=_get_api_key())
   184	        self._prompt_template = prompt_template or load_prompt_template(Path("Prompt.md"))
   185
   186	    @property
   187	    def name(self) -> str:
   188	        return f"gemini:{self.config.model}"
   189
   190	    def transcribe(self, audio_path: Path, language: str | None = None) -> ASRResult:
   191	        language_code = language
   192	        prompt_language = to_prompt_language_hint(language_code)
   193	        prompt = self._prompt_template.replace("<language>", prompt_language)
   194	        audio_bytes = audio_path.read_bytes()
   195	        mime_type = _mime_type_for_path(audio_path)
   196
   197	        cfg = self._types.GenerateContentConfig(
   198	            temperature=self.config.temperature,
   199	            max_output_tokens=self.config.max_output_tokens,
   200	            response_mime_type=self.config.response_mime_type,
   201	        )
   202
   203	        last_err: Exception | None = None
   204	        for attempt in range(self.config.retries + 1):
   205	            try:
   206	                response = self._client.models.generate_content(
   207	                    model=self.config.model,
   208	                    contents=[
   209	                        self._types.Part.from_text(text=prompt),
   210	                        self._types.Part.from_bytes(data=audio_bytes, mime_type=mime_type),
   211	                    ],
   212	                    config=cfg,
   213	                )
   214	                raw_text = getattr(response, "text", None) or ""
   215	                parsed = parse_strict_output(raw_text)
   216	                transcript = parsed.transcript
   217
   218	                # Normalization is performed later in the pipeline; keep the raw line.
   219	                return ASRResult(
   220	                    model=self.name,
   221	                    text=transcript,
   222	                    language=language_code,
   223	                    language_probability=None,
   224	                    segments=[],
   225	                    meta={
   226	                        "mime_type": mime_type,
   227	                        "temperature": self.config.temperature,
   228	                        "max_output_tokens": self.config.max_output_tokens,
   229	                        "response_mime_type": self.config.response_mime_type,
   230	                        "prompt_language_hint": prompt_language,
   231	                        "language_code": language_code,
   232	                        "raw_output": raw_text,
   233	                        "parse": {
   234	                            "had_label": parsed.had_label,
   235	                            "joined_lines": parsed.joined_lines,
   236	                            "stripped_punctuation": parsed.stripped_punctuation,
   237	                            "extra_outputs_present": parsed.extra_outputs_present,
   238	                        },
   239	                    },
   240	                )
   241	            except Exception as e:
   242	                last_err = e
   243	                if attempt >= self.config.retries:
   244	                    break
   245	                backoff = self.config.initial_backoff_sec * (2**attempt)
   246	                time.sleep(backoff)
   247
   248	        raise RuntimeError(f"gemini_transcribe_failed: {type(last_err).__name__}: {last_err}")
   249
   250
   251	def safe_normalized_transcript(text: str, lang: str | None) -> str:
   252	    """
   253	    Convenience helper: strict normalize for alignment/gating.
   254	    """
   255	    return normalize_text(text, lang=lang)


nl -ba maya_transcribe/verify.py | sed -n '1,260p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	from dataclasses import dataclass
     4	from typing import Any
     5
     6	from maya_transcribe.lid import LIDResult
     7
     8
     9	@dataclass(frozen=True)
    10	class Verification:
    11	    decision: str  # "accept" | "human" | "reject"
    12	    reasons: list[str]
    13	    scores: dict[str, Any]
    14
    15
    16	def lid_confidence(lid: LIDResult | None) -> dict[str, Any]:
    17	    if not lid or not lid.predictions:
    18	        return {"available": False, "top1": None, "top1_prob": None, "margin": None, "confident": False}
    19	    top1 = lid.predictions[0]
    20	    top2_prob = lid.predictions[1].prob if len(lid.predictions) > 1 else 0.0
    21	    margin = float(top1.prob) - float(top2_prob)
    22	    return {
    23	        "available": True,
    24	        "top1": top1.code,
    25	        "top1_prob": float(top1.prob),
    26	        "margin": margin,
    27	        "confident": False,
    28	    }
    29
    30
    31	def verify_segment(
    32	    *,
    33	    qc: dict[str, Any],
    34	    lid: LIDResult | None,
    35	    final_text: str,
    36	    rover: dict[str, Any],
    37	    asr_summaries: dict[str, Any],
    38	    primary_asr_key: str | None = None,
    39	    ctc: dict[str, Any] | None = None,
    40	    thresholds: dict[str, Any] | None = None,
    41	) -> Verification:
    42	    """
    43	    Conservative verification gate (precision-first).
    44	    """
    45	    thresholds = thresholds or {}
    46	    min_duration = float(thresholds.get("min_duration_sec", 0.8))
    47	    max_clipping = float(thresholds.get("max_clipping_fraction", 0.01))
    48	    max_disagreement = float(thresholds.get("max_rover_disagreement_rate", 0.03))
    49	    lid_min_prob = float(thresholds.get("lid_min_prob", 0.90))
    50	    lid_min_margin = float(thresholds.get("lid_min_margin", 0.20))
    51	    require_lid = bool(thresholds.get("require_lid", True))
    52
    53	    # Whisper-style failure heuristics (see OpenAI whisper fallback logic).
    54	    min_avg_logprob = float(thresholds.get("min_avg_logprob_mean", -1.0))
    55	    max_compression_ratio = float(thresholds.get("max_compression_ratio_mean", 2.4))
    56
    57	    # CTC fit (audio-grounded) gating.
    58	    require_ctc = bool(thresholds.get("require_ctc", False))
    59	    min_ctc_margin = float(thresholds.get("min_ctc_margin", 0.0))
    60	    min_ctc_token_coverage = float(thresholds.get("min_ctc_token_coverage", 0.85))
    61	    min_ctc_token_score_mean = float(thresholds.get("min_ctc_token_score_mean", -3.0))
    62	    fail_on_ctc_span_mismatch = bool(thresholds.get("fail_on_ctc_span_mismatch", True))
    63
    64	    reasons: list[str] = []
    65	    decision = "accept"
    66
    67	    duration = float(qc.get("duration_sec") or 0.0)
    68	    clipping = float(qc.get("clipping_fraction") or 0.0)
    69	    if duration < min_duration:
    70	        decision = "reject"
    71	        reasons.append("too_short")
    72	    if clipping > max_clipping:
    73	        decision = "human" if decision != "reject" else decision
    74	        reasons.append("clipping")
    75
    76	    lidc = lid_confidence(lid)
    77	    if lidc["available"]:
    78	        lidc["confident"] = bool(lidc["top1_prob"] >= lid_min_prob and lidc["margin"] >= lid_min_margin)
    79	        if not lidc["confident"]:
    80	            decision = "human" if decision != "reject" else decision
    81	            reasons.append("lid_uncertain")
    82	    else:
    83	        reasons.append("lid_missing")
    84	        if require_lid:
    85	            decision = "human" if decision != "reject" else decision
    86
    87	    disagreement_rate = float(rover.get("disagreement_rate") or 0.0)
    88	    if disagreement_rate > max_disagreement:
    89	        decision = "human" if decision != "reject" else decision
    90	        reasons.append("asr_disagreement")
    91
    92	    final_text = (final_text or "").strip()
    93	    if "<unk>" in final_text or "<inaudible>" in final_text:
    94	        decision = "human" if decision != "reject" else decision
    95	        reasons.append("contains_unk")
    96
    97	    # Whisper-style safety: flag if any decoder claims "no speech" with high probability.
    98	    no_speech_probs = []
    99	    for summary in asr_summaries.values():
   100	        p = summary.get("no_speech_prob_mean")
   101	        if p is None:
   102	            continue
   103	        no_speech_probs.append(float(p))
   104	    if no_speech_probs and max(no_speech_probs) > 0.8:
   105	        decision = "reject"
   106	        reasons.append("no_speech")
   107
   108	    # Whisper-style hallucination heuristics: apply to the primary ASR (or rover skeleton fallback).
   109	    asr_key_for_heuristics = primary_asr_key or str(rover.get("skeleton_key") or "")
   110	    heur = asr_summaries.get(asr_key_for_heuristics, {}) if asr_key_for_heuristics else {}
   111	    avg_lp = heur.get("avg_logprob_mean")
   112	    comp = heur.get("compression_ratio_mean")
   113	    if avg_lp is not None and float(avg_lp) < min_avg_logprob:
   114	        decision = "human" if decision != "reject" else decision
   115	        reasons.append("low_logprob")
   116	    if comp is not None and float(comp) > max_compression_ratio:
   117	        decision = "human" if decision != "reject" else decision
   118	        reasons.append("high_compression_ratio")
   119
   120	    # Audio-grounded fit check (when available).
   121	    if require_ctc and (ctc is None or ctc.get("best_loss") is None):
   122	        decision = "human" if decision != "reject" else decision
   123	        reasons.append("ctc_missing")
   124	    if ctc is not None:
   125	        margin = ctc.get("margin")
   126	        if margin is not None and float(margin) < min_ctc_margin:
   127	            decision = "human" if decision != "reject" else decision
   128	            reasons.append("ctc_ambiguous")
   129
   130	        best_key = ctc.get("best_key")
   131	        scores = ctc.get("scores")
   132	        if isinstance(best_key, str) and isinstance(scores, dict):
   133	            best = scores.get(best_key) or {}
   134	            if isinstance(best, dict):
   135	                cov = best.get("token_coverage")
   136	                if cov is not None and float(cov) < min_ctc_token_coverage:
   137	                    decision = "human" if decision != "reject" else decision
   138	                    reasons.append("ctc_low_coverage")
   139	                mean_tok = best.get("token_score_mean")
   140	                if mean_tok is not None and float(mean_tok) < min_ctc_token_score_mean:
   141	                    decision = "human" if decision != "reject" else decision
   142	                    reasons.append("ctc_low_token_score")
   143	                if fail_on_ctc_span_mismatch and bool(best.get("span_mismatch")):
   144	                    decision = "human" if decision != "reject" else decision
   145	                    reasons.append("ctc_span_mismatch")
   146
   147	    scores = {
   148	        "duration_sec": duration,
   149	        "clipping_fraction": clipping,
   150	        "rover_disagreement_rate": disagreement_rate,
   151	        "lid": lidc,
   152	        "no_speech_prob_max": max(no_speech_probs) if no_speech_probs else None,
   153	        "heuristics_asr_key": asr_key_for_heuristics or None,
   154	        "avg_logprob_mean": avg_lp,
   155	        "compression_ratio_mean": comp,
   156	        "ctc": ctc,
   157	    }
   158
   159	    return Verification(decision=decision, reasons=reasons, scores=scores)


nl -ba maya_transcribe/eval.py | sed -n '1,360p'


Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 287
Output:
     1	from __future__ import annotations
     2
     3	import json
     4	from dataclasses import dataclass
     5	from pathlib import Path
     6	from typing import Any
     7
     8	from jiwer import cer, wer
     9
    10	from maya_transcribe.io import read_jsonl
    11	from maya_transcribe.textnorm import normalize_text
    12
    13
    14	@dataclass(frozen=True)
    15	class EvalRow:
    16	    segment_id: str
    17	    language: str | None
    18	    decision: str | None
    19	    duration_sec: float | None
    20	    duration_bucket: str | None
    21	    ref: str
    22	    hyp: str
    23	    wer: float
    24	    cer: float
    25	    oracle_best_wer: float | None = None
    26	    oracle_best_cer: float | None = None
    27	    oracle_best_key: str | None = None
    28	    selection_gap_wer: float | None = None
    29
    30
    31	def load_refs(refs_path: Path) -> dict[str, dict[str, Any]]:
    32	    """
    33	    Read a JSONL file of references.
    34	    Expected keys (any one): `text`, `transcript`, `reference`.
    35	    Required key: `segment_id` (or `id`).
    36	    """
    37	    refs: dict[str, dict[str, Any]] = {}
    38	    for row in read_jsonl(refs_path):
    39	        segment_id = str(row.get("segment_id") or row.get("id") or "").strip()
    40	        if not segment_id:
    41	            continue
    42	        refs[segment_id] = row
    43	    return refs
    44
    45
    46	def _get_text(row: dict[str, Any], keys: tuple[str, ...]) -> str | None:
    47	    for k in keys:
    48	        v = row.get(k)
    49	        if isinstance(v, str) and v.strip():
    50	            return v
    51	    return None
    52
    53
    54	def evaluate(
    55	    results_path: Path,
    56	    refs_path: Path,
    57	    out_path: Path | None = None,
    58	    hypothesis: str = "final",
    59	    oracle: bool = True,
    60	) -> dict[str, Any]:
    61	    refs = load_refs(refs_path)
    62	    results = read_jsonl(results_path)
    63
    64	    rows: list[EvalRow] = []
    65	    for r in results:
    66	        segment_id = str(r.get("segment_id") or "").strip()
    67	        if not segment_id or segment_id not in refs:
    68	            continue
    69
    70	        ref_row = refs[segment_id]
    71	        ref_text = _get_text(ref_row, ("text", "transcript", "reference"))
    72	        if ref_text is None:
    73	            continue
    74
    75	        hyp_text = extract_hypothesis_text(r, hypothesis=hypothesis)
    76	        if hyp_text is None:
    77	            continue
    78
    79	        lang = None
    80	        route = r.get("route") or {}
    81	        if isinstance(route, dict):
    82	            lang = route.get("language_hint")
    83
    84	        decision = None
    85	        ver = r.get("verification") or {}
    86	        if isinstance(ver, dict):
    87	            decision = ver.get("decision")
    88
    89	        duration_sec = None
    90	        qc = r.get("qc") or {}
    91	        if isinstance(qc, dict):
    92	            try:
    93	                duration_sec = float(qc.get("duration_sec")) if qc.get("duration_sec") is not None else None
    94	            except (TypeError, ValueError):
    95	                duration_sec = None
    96	        bucket = duration_bucket(duration_sec)
    97
    98	        ref_norm = normalize_text(ref_text, lang=lang)
    99	        hyp_norm = normalize_text(hyp_text, lang=lang)
   100	        w = float(wer(ref_norm, hyp_norm))
   101	        c = float(cer(ref_norm.replace(" ", ""), hyp_norm.replace(" ", "")))
   102
   103	        oracle_best_wer = None
   104	        oracle_best_cer = None
   105	        oracle_best_key = None
   106	        selection_gap_wer = None
   107
   108	        if oracle:
   109	            cand_wer: dict[str, float] = {}
   110	            cand_cer: dict[str, float] = {}
   111	            for key, text in extract_all_candidates(result_row=r).items():
   112	                ref_n = ref_norm
   113	                hyp_n = normalize_text(text, lang=lang)
   114	                cand_wer[key] = float(wer(ref_n, hyp_n))
   115	                cand_cer[key] = float(cer(ref_n.replace(" ", ""), hyp_n.replace(" ", "")))
   116
   117	            if cand_wer:
   118	                oracle_best_key, oracle_best_wer = min(cand_wer.items(), key=lambda kv: kv[1])
   119	                oracle_best_cer = cand_cer.get(oracle_best_key)
   120	                selection_gap_wer = float(w) - float(oracle_best_wer)
   121
   122	        rows.append(
   123	            EvalRow(
   124	                segment_id=segment_id,
   125	                language=lang,
   126	                decision=str(decision) if decision is not None else None,
   127	                duration_sec=duration_sec,
   128	                duration_bucket=bucket,
   129	                ref=ref_text,
   130	                hyp=hyp_text,
   131	                wer=w,
   132	                cer=c,
   133…2455 chars truncated…ws if r.wer > 0.10) / len(rows)) if rows else None,
   177	        "wer_by_language_mean": {k: sum(v) / len(v) for k, v in sorted(by_lang.items())},
   178	        "wer_by_decision_mean": {k: sum(v) / len(v) for k, v in sorted(by_dec.items())},
   179	        "wer_by_duration_bucket_mean": {k: sum(v) / len(v) for k, v in sorted(by_bucket.items())},
   180	        "cer_by_language_mean": {k: sum(v) / len(v) for k, v in sorted(by_lang_cer.items())},
   181	        "cer_by_decision_mean": {k: sum(v) / len(v) for k, v in sorted(by_dec_cer.items())},
   182	        "cer_by_duration_bucket_mean": {k: sum(v) / len(v) for k, v in sorted(by_bucket_cer.items())},
   183	        "oracle": {
   184	            "enabled": bool(oracle),
   185	            "oracle_best_wer_mean": (sum(oracle_best_wers) / len(oracle_best_wers)) if oracle_best_wers else None,
   186	            "selection_gap_wer_mean": (sum(selection_gaps) / len(selection_gaps)) if selection_gaps else None,
   187	            "selection_gap_wer_p90": _percentile(selection_gaps, 0.90),
   188	        },
   189	    }
   190
   191	    if out_path is not None:
   192	        out_path.parent.mkdir(parents=True, exist_ok=True)
   193	        out_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
   194
   195	    return summary
   196
   197
   198	def extract_hypothesis_text(result_row: dict[str, Any], hypothesis: str) -> str | None:
   199	    """
   200	    hypothesis:
   201	      - "final": uses result_row["final"]["text"] (falls back to rover if missing)
   202	      - "rover" (default): uses result_row["rover"]["text"]
   203	      - "asr:<key>": uses result_row["asr"][<key>]["text"]
   204	    """
   205	    hypothesis = (hypothesis or "rover").strip()
   206	    if hypothesis == "final":
   207	        final = result_row.get("final") or {}
   208	        if isinstance(final, dict) and isinstance(final.get("text"), str):
   209	            return final["text"]
   210	        hypothesis = "rover"
   211	    if hypothesis == "rover":
   212	        rover = result_row.get("rover") or {}
   213	        if isinstance(rover, dict) and isinstance(rover.get("text"), str):
   214	            return rover["text"]
   215	        return None
   216
   217	    if hypothesis.startswith("asr:"):
   218	        key = hypothesis[len("asr:") :].strip()
   219	        asr = result_row.get("asr") or {}
   220	        if isinstance(asr, dict):
   221	            entry = asr.get(key) or {}
   222	            if isinstance(entry, dict) and isinstance(entry.get("text"), str):
   223	                return entry["text"]
   224	        return None
   225
   226	    return None
   227
   228
   229	def extract_all_candidates(result_row: dict[str, Any]) -> dict[str, str]:
   230	    """
   231	    Collect all text candidates available in a pipeline result row.
   232	    Keys are stable and namespaced (final/rover/asr:<key>).
   233	    """
   234	    candidates: dict[str, str] = {}
   235
   236	    final = extract_hypothesis_text(result_row, "final")
   237	    if final is not None:
   238	        candidates["final"] = final
   239
   240	    rover = extract_hypothesis_text(result_row, "rover")
   241	    if rover is not None:
   242	        candidates["rover"] = rover
   243
   244	    asr = result_row.get("asr") or {}
   245	    if isinstance(asr, dict):
   246	        for key, entry in asr.items():
   247	            if not isinstance(key, str):
   248	                continue
   249	            if not isinstance(entry, dict):
   250	                continue
   251	            text = entry.get("text")
   252	            if isinstance(text, str) and text.strip():
   253	                candidates[f"asr:{key}"] = text
   254
   255	    # Also include normalized hypotheses if present (useful for debugging).
   256	    normalized = result_row.get("normalized") or {}
   257	    if isinstance(normalized, dict):
   258	        for key, text in normalized.items():
   259	            if not isinstance(key, str) or not isinstance(text, str):
   260	                continue
   261	            if text.strip():
   262	                candidates[f"norm:{key}"] = text
   263
   264	    return candidates
   265
   266
   267	def duration_bucket(duration_sec: float | None) -> str | None:
   268	    if duration_sec is None:
   269	        return None
   270	    if duration_sec < 3.0:
   271	        return "0-3s"
   272	    if duration_sec < 10.0:
   273	        return "3-10s"
   274	    if duration_sec < 30.0:
   275	        return "10-30s"
   276	    if duration_sec < 60.0:
   277	        return "30-60s"
   278	    return "60s+"
   279
   280
   281	def _percentile(xs: list[float], q: float) -> float | None:
   282	    if not xs:
   283	        return None
   284	    xs_sorted = sorted(xs)
   285	    idx = int(round((len(xs_sorted) - 1) * q))
   286	    idx = max(0, min(len(xs_sorted) - 1, idx))
   287	    return float(xs_sorted[idx])


nl -ba maya_transcribe/pipeline.py | sed -n '120,340p'


Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 221
Output:
   120
   121	    results_path = out_dir / "results.jsonl"
   122	    summary_path = out_dir / "summary.json"
   123
   124	    counts = {"accept": 0, "human": 0, "reject": 0, "error": 0}
   125	    timings_total: list[float] = []
   126
   127	    with results_path.open("w", encoding="utf-8") as f:
   128	        for src in track(audio_files, description="Transcribing"):
   129	            ident = segment_identity(src, root=segments_dir)
   130	            segment_id = ident.segment_id
   131	            t0 = time.perf_counter()
   132
   133	            try:
   134	                # 1) Preprocess
   135	                t_pre0 = time.perf_counter()
   136	                src_info = ffprobe(src)
   137	                wav_path = out_dir / "audio" / f"{segment_id}.wav"
   138	                normalize_audio(
   139	                    src,
   140	                    wav_path,
   141	                    trim_silence=trim_silence,
   142	                    silence_threshold_db=silence_threshold_db,
   143	                    min_silence_sec=min_silence_sec,
   144	                )
   145	                info = ffprobe(wav_path)
   146	                qc = compute_qc(wav_path)
   147	                t_pre1 = time.perf_counter()
   148
   149	                # 2) LID
   150	                t_lid0 = time.perf_counter()
   151	                lid: LIDResult | None = None
   152	                if lid_backend is not None:
   153	                    lid = lid_backend.predict(wav_path, top_k=5)
   154	                t_lid1 = time.perf_counter()
   155
   156	                # Route language hint (only if confident).
   157	                route_lang = None
   158	                if lid and lid.predictions:
   159	                    top1 = lid.predictions[0]
   160	                    top2p = lid.predictions[1].prob if len(lid.predictions) > 1 else 0.0
   161	                    if float(top1.prob) >= float(thresholds.get("lid_min_prob", 0.90)) and (
   162	                        float(top1.prob) - float(top2p)
   163	                    ) >= float(thresholds.get("lid_min_margin", 0.20)):
   164	                        route_lang = top1.code
   165
   166	                # 3) ASR ensemble
   167	                t_asr0 = time.perf_counter()
   168	                asr_results: dict[str, ASRResult] = {}
   169
   170	                # Whisper: deterministic + a small sampling diversity pass.
   171	                det = whisper_backend.transcribe(wav_path, language=route_lang, beam_size=5, best_of=5, temperature=0.0)
   172	                primary_asr_key = f"{det.model}|det"
   173	                asr_results[primary_asr_key] = det
   174	                samp = whisper_backend.transcribe(
   175	                    wav_path, language=route_lang, beam_size=5, best_of=10, temperature=0.2
   176	                )
   177	                asr_results[f"{samp.model}|samp_t0.2"] = samp
   178
   179	                for tf_backend in transformers_backends:
   180	                    res = tf_backend.transcribe(wav_path, language=route_lang, num_beams=5, max_new_tokens=256)
   181	                    asr_results[res.model] = res
   182
   183	                # Gemini: optional extra hypothesis (prefer running only on "hard" segments).
   184	                if gemini_backend is not None:
   185	                    run_gemini = True
   186	                    if gemini_hard_only:
   187	                        det_sum = _asr_summary(det)
   188	                        avg_lp = det_sum.get("avg_logprob_mean")
   189	                        comp = det_sum.get("compression_ratio_mean")
   190	                        no_speech = det_sum.get("no_speech_prob_mean")
   191	                        run_gemini = bool(
   192	                            route_lang is None
   193	                            or (no_speech is not None and float(no_speech) > 0.5)
   194	                            or (avg_lp is not None and float(avg_lp) < float(thresholds.get("min_avg_logprob_mean", -1.0)))
   195	                            or (
   196	                                comp is not None
   197	                                and float(comp) > float(thresholds.get("max_compression_ratio_mean", 2.4))
   198	                            )
   199	                        )
   200
   201	                    if run_gemini:
   202	                        try:
   203	                            g = gemini_backend.transcribe(wav_path, language=route_lang)
   204	                            asr_results[g.model] = g
   205	                        except Exception:
   206	                            pass
   207
   208	                t_asr1 = time.perf_counter()
   209
   210	                ensemble_models = sorted({v.model for v in asr_results.values()})
   211	                ensemble_backends = sorted({v.model.split(":", 1)[0] for v in asr_results.values()})
   212	                ensemble_warnings: list[str] = []
   213	                i…1822 chars truncated…r_summary(v) for k, v in asr_results.items()}
   246	                verification = verify_segment(
   247	                    qc=asdict(qc),
   248	                    lid=lid,
   249	                    final_text=final_text,
   250	                    rover={
   251	                        "text": rover.text,
   252	                        "disagreement_rate": rover.disagreement_rate,
   253	                        "skeleton_key": rover.skeleton_key,
   254	                    },
   255	                    asr_summaries=asr_summaries,
   256	                    primary_asr_key=primary_asr_key,
   257	                    ctc=ctc_info,
   258	                    thresholds=thresholds,
   259	                )
   260	                t_ver1 = time.perf_counter()
   261
   262	                record = {
   263	                    "schema_version": 1,
   264	                    "segment_id": segment_id,
   265	                    "base_id": ident.base_id,
   266	                    "relpath": ident.relpath,
   267	                    "uid": ident.uid,
   268	                    "src_path": str(src),
   269	                    "wav_path": str(wav_path),
   270	                    "src_ffprobe": asdict(src_info),
   271	                    "ffprobe": asdict(info),
   272	                    "qc": asdict(qc),
   273	                    "lid": (
   274	                        None
   275	                        if lid is None
   276	                        else {
   277	                            "model": lid.model,
   278	                            "predictions": [{"code": p.code, "prob": p.prob} for p in lid.predictions],
   279	                        }
   280	                    ),
   281	                    "route": {"language_hint": route_lang},
   282	                    "asr": {
   283	                        k: {
   284	                            "model": v.model,
   285	                            "text": v.text,
   286	                            "language": v.language,
   287	                            "language_probability": v.language_probability,
   288	                            "segments": [asdict(s) for s in v.segments],
   289	                            "meta": v.meta,
   290	                        }
   291	                        for k, v in asr_results.items()
   292	                    },
   293	                    "asr_summary": asr_summaries,
   294	                    "ensemble": {
   295	                        "unique_models": ensemble_models,
   296	                        "unique_backends": ensemble_backends,
   297	                        "warnings": ensemble_warnings,
   298	                    },
   299	                    "normalized": norm_hyps,
   300	                    "rover": {
   301	                        "text": rover.text,
   302	                        "skeleton_key": rover.skeleton_key,
   303	                        "disagreement_rate": rover.disagreement_rate,
   304	                        "disagreements": [
   305	                            {"kind": d.kind, "index": d.index, "chosen": d.chosen, "options": d.options}
   306	                            for d in rover.disagreements
   307	                        ],
   308	                    },
   309	                    "ctc": ctc_info,
   310	                    "final": {"text": final_text, "source": final_source},
   311	                    "verification": {
   312	                        "decision": verification.decision,
   313	                        "reasons": verification.reasons,
   314	                        "scores": verification.scores,
   315	                    },
   316	                    "config": {
   317	                        "trim_silence": trim_silence,
   318	                        "silence_threshold_db": silence_threshold_db,
   319	                        "min_silence_sec": min_silence_sec,
   320	                        "gemini_model": gemini_model,
   321	                        "gemini_hard_only": gemini_hard_only,
   322	                        "ctc_model": ctc_model,
   323	                    },
   324	                    "timing": {
   325	                        "preprocess_sec": t_pre1 - t_pre0,
   326	                        "lid_sec": t_lid1 - t_lid0,
   327	                        "asr_sec": t_asr1 - t_asr0,
   328	                        "normalize_sec": t_norm1 - t_norm0,
   329	                        "rover_sec": t_rover1 - t_rover0,
   330	                        "ctc_sec": t_ctc1 - t_ctc0,
   331	                        "verify_sec": t_ver1 - t_ver0,
   332	                        "total_sec": time.perf_counter() - t0,
   333	                    },
   334	                }
   335
   336	                f.write(json.dumps(record, ensure_ascii=False) + "\n")
   337	                decision = verification.decision
   338	                counts[decision] = counts.get(decision, 0) + 1
   339	                timings_total.append(record["timing"]["total_sec"])
   340


nl -ba maya_transcribe/pipeline.py | sed -n '200,280p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   200
   201	                    if run_gemini:
   202	                        try:
   203	                            g = gemini_backend.transcribe(wav_path, language=route_lang)
   204	                            asr_results[g.model] = g
   205	                        except Exception:
   206	                            pass
   207
   208	                t_asr1 = time.perf_counter()
   209
   210	                ensemble_models = sorted({v.model for v in asr_results.values()})
   211	                ensemble_backends = sorted({v.model.split(":", 1)[0] for v in asr_results.values()})
   212	                ensemble_warnings: list[str] = []
   213	                if len(ensemble_models) < 2:
   214	                    ensemble_warnings.append("low_model_diversity")
   215
   216	                # 4) Normalize hypotheses
   217	                t_norm0 = time.perf_counter()
   218	                norm_hyps: dict[str, str] = {}
   219	                for k, res in asr_results.items():
   220	                    lang_hint = res.language or route_lang
   221	                    norm_hyps[k] = normalize_text(res.text, lang=lang_hint)
   222	                t_norm1 = time.perf_counter()
   223
   224	                # 5) ROVER
   225	                t_rover0 = time.perf_counter()
   226	                rover = rover_combine(norm_hyps)
   227	                t_rover1 = time.perf_counter()
   228
   229	                # 5.5) Audio-grounded fit scoring (optional): choose best candidate by CTC loss.
   230	                t_ctc0 = time.perf_counter()
   231	                ctc_info = None
   232	                final_text = rover.text
   233	                final_source = "rover"
   234	                if ctc_scorer is not None and rank_by_ctc is not None:
   235	                    candidates = {"rover": rover.text, **norm_hyps}
   236	                    ctc_info = rank_by_ctc(ctc_scorer, wav_path, candidates=candidates, lang=route_lang)
   237	                    best_key = ctc_info.get("best_key") if isinstance(ctc_info, dict) else None
   238	                    if best_key and best_key in candidates:
   239	                        final_text = candidates[best_key]
   240	                        final_source = "rover" if best_key == "rover" else f"asr:{best_key}"
   241	                t_ctc1 = time.perf_counter()
   242
   243	                # 6) Verify
   244	                t_ver0 = time.perf_counter()
   245	                asr_summaries = {k: _asr_summary(v) for k, v in asr_results.items()}
   246	                verification = verify_segment(
   247	                    qc=asdict(qc),
   248	                    lid=lid,
   249	                    final_text=final_text,
   250	                    rover={
   251	                        "text": rover.text,
   252	                        "disagreement_rate": rover.disagreement_rate,
   253	                        "skeleton_key": rover.skeleton_key,
   254	                    },
   255	                    asr_summaries=asr_summaries,
   256	                    primary_asr_key=primary_asr_key,
   257	                    ctc=ctc_info,
   258	                    thresholds=thresholds,
   259	                )
   260	                t_ver1 = time.perf_counter()
   261
   262	                record = {
   263	                    "schema_version": 1,
   264	                    "segment_id": segment_id,
   265	                    "base_id": ident.base_id,
   266	                    "relpath": ident.relpath,
   267	                    "uid": ident.uid,
   268	                    "src_path": str(src),
   269	                    "wav_path": str(wav_path),
   270	                    "src_ffprobe": asdict(src_info),
   271	                    "ffprobe": asdict(info),
   272	                    "qc": asdict(qc),
   273	                    "lid": (
   274	                        None
   275	                        if lid is None
   276	                        else {
   277	                            "model": lid.model,
   278	                            "predictions": [{"code": p.code, "prob": p.prob} for p in lid.predictions],
   279	                        }
   280	                    ),


**Checking pipeline details**

I need to look into the pipeline signature and how the lid backend is being instantiated. It used to have `require_lid_backend`, but now it seems there's something with `stable segment_id`. I’ll check the top portion of `run_pipeline` to confirm these details regarding `require_lid_backend` and other relevant lines. This will help clarify how everything is structured in those first 120 lines. Let's get into it!


nl -ba maya_transcribe/pipeline.py | sed -n '1,130p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	import json
     4	import time
     5	from dataclasses import asdict
     6	from pathlib import Path
     7	from typing import Any
     8
     9	from rich.progress import track
    10
    11	from maya_transcribe.asr import ASRResult, FasterWhisperASR, TransformersWhisperASR
    12	from maya_transcribe.audio import compute_qc, ffprobe, normalize_audio
    13	from maya_transcribe.ids import segment_identity
    14	from maya_transcribe.lid import LIDResult, MMSLID
    15	from maya_transcribe.rover import rover_combine
    16	from maya_transcribe.textnorm import normalize_text
    17	from maya_transcribe.verify import verify_segment
    18
    19
    20	def _iter_audio_files(segments_dir: Path) -> list[Path]:
    21	    audio_files: list[Path] = []
    22	    for ext in ("*.wav", "*.mp3", "*.m4a", "*.flac", "*.ogg", "*.opus"):
    23	        audio_files.extend(sorted(segments_dir.rglob(ext)))
    24	    return sorted(set(audio_files))
    25
    26
    27	def _asr_summary(result: ASRResult) -> dict[str, Any]:
    28	    avg_logprobs = [s.avg_logprob for s in result.segments if s.avg_logprob is not None]
    29	    no_speech_probs = [s.no_speech_prob for s in result.segments if s.no_speech_prob is not None]
    30	    compression_ratios = [s.compression_ratio for s in result.segments if s.compression_ratio is not None]
    31
    32	    def mean(xs: list[float]) -> float | None:
    33	        return (sum(xs) / len(xs)) if xs else None
    34
    35	    return {
    36	        "language": result.language,
    37	        "language_probability": result.language_probability,
    38	        "avg_logprob_mean": mean([float(x) for x in avg_logprobs]),
    39	        "no_speech_prob_mean": mean([float(x) for x in no_speech_probs]),
    40	        "compression_ratio_mean": mean([float(x) for x in compression_ratios]),
    41	    }
    42
    43
    44	def run_pipeline(
    45	    *,
    46	    segments_dir: Path,
    47	    out_dir: Path,
    48	    lid_model_id: str = "facebook/mms-lid-256",
    49	    whisper_model: str = "large-v3",
    50	    extra_transformers_whisper_models: list[str] | None = None,
    51	    gemini_model: str | None = None,
    52	    gemini_prompt_path: Path | None = None,
    53	    gemini_max_output_tokens: int = 512,
    54	    gemini_hard_only: bool = True,
    55	    ctc_model: str | None = None,
    56	    ctc_model_map: dict[str, str] | None = None,
    57	    require_lid_backend: bool = True,
    58	    trim_silence: bool = True,
    59	    silence_threshold_db: float = -40.0,
    60	    min_silence_sec: float = 0.15,
    61	    limit: int | None = None,
    62	    thresholds: dict[str, Any] | None = None,
    63	) -> Path:
    64	    """
    65	    End-to-end run: preprocess -> LID -> ASR ensemble -> normalize -> ROVER -> verify.
    66	    Writes `out_dir/results.jsonl` and returns that path.
    67	    """
    68	    thresholds = thresholds or {}
    69	    if (ctc_model or ctc_model_map) and "require_ctc" not in thresholds:
    70	        thresholds["require_ctc"] = True
    71	    if (ctc_model or ctc_model_map) and "min_ctc_margin" not in thresholds:
    72	        thresholds["min_ctc_margin"] = 0.05
    73	    out_dir.mkdir(parents=True, exist_ok=True)
    74
    75	    audio_files = _iter_audio_files(segments_dir)
    76	    if not audio_files:
    77	        raise RuntimeError(f"No audio files found under: {segments_dir}")
    78	    if limit is not None:
    79	        audio_files = audio_files[: max(0, int(limit))]
    80
    81	    # Instantiate models once.
    82	    lid_backend: MMSLID | None
    83	    try:
    84	        lid_backend = MMSLID(model_id=lid_model_id)
    85	    except Exception as e:
    86	        if require_lid_backend:
    87	            raise RuntimeError(
    88	                "Failed to initialize MMS-LID. Install torch/transformers deps (pip install -e '.[torch]') "
    89	                "or run with require_lid_backend=False."
    90	            ) from e
    91	        lid_backend = None
    92
    93	    whisper_backend = FasterWhisperASR(model_size_or_path=whisper_model)
    94
    95	    extra_transformers_whisper_models = extra_transformers_whisper_models or []
    96	    transformers_backends: list[TransformersWhisperASR] = []
    97	    for model_id in extra_transformers_whisper_models:
    98	        transformers_backends.append(TransformersWhisperASR(model_id=model_id))
    99
   100	    ctc_scorer = None
   101	    rank_by_ctc = None
   102	    if ctc_model or ctc_model_map:
   103	        from maya_transcribe.ctc_fit import CTCFitScorer, rank_candidates_by_ctc
   104
   105	        ctc_scorer = CTCFitScorer(model_map=ctc_model_map, default_model=ctc_model)
   106	        rank_by_ctc = rank_candidates_by_ctc
   107
   108	    gemini_backend = None
   109	    if gemini_model:
   110	        try:
   111	            from maya_transcribe.gemini_asr import GeminiASR, GeminiConfig, load_prompt_template
   112
   113	            prompt = load_prompt_template(gemini_prompt_path or Path("Prompt.md"))
   114	            gemini_backend = GeminiASR(
   115	                config=GeminiConfig(model=gemini_model, max_output_tokens=gemini_max_output_tokens),
   116	                prompt_template=prompt,
   117	            )
   118	        except Exception as e:
   119	            raise RuntimeError("Failed to initialize Gemini backend (check GEMINI_KEY and google-genai).") from e
   120
   121	    results_path = out_dir / "results.jsonl"
   122	    summary_path = out_dir / "summary.json"
   123
   124	    counts = {"accept": 0, "human": 0, "reject": 0, "error": 0}
   125	    timings_total: list[float] = []
   126
   127	    with results_path.open("w", encoding="utf-8") as f:
   128	        for src in track(audio_files, description="Transcribing"):
   129	            ident = segment_identity(src, root=segments_dir)
   130	            segment_id = ident.segment_id


**Searching for WER and AI4Bharat**

I’m thinking about expected WER ranges, but I want to include a disclaimer when I provide them. I’ll look for the AI4Bharat Vistaar paper and the related dataset names. I can fetch the arXiv abstract, but WER tables might be challenging to parse from PDFs. It seems some WER ranges for IndicWhisper are already referenced in a previous source. I’ll use `curl` for a targeted internet search, possibly looking up URLs directly.


**Fetching Vistaar Dataset Info**

I'm planning to start by checking the arXiv page for the Vistaar paper. This page might have useful details like dataset names, languages, and evaluation set names. To get this information, I’ll use the command `curl -L https://arxiv.org/abs/2305.15386` to access the page and see what I can find about the datasets. It's a straightforward step, but I hope it yields the details I need!


python3 - <<'PY'
import re, textwrap, sys
import requests
url='https://arxiv.org/abs/2305.15386'
html=requests.get(url,timeout=20).text
# extract title and abstract
m=re.search(r'
\s*Title:\s*(.*?)\s*
', html, re.S)
if m:
    title=re.sub('<.*?>','',m.group(1)).strip()
    print('Title:',title)

m=re.search(r'
\s*Abstract:(.*?)
', html, re.S)
if m:
    abs_txt=re.sub('<.*?>','',m.group(1))
    abs_txt=' '.join(abs_txt.split())
    print('\nAbstract (first 600 chars):')
    print(textwrap.fill(abs_txt[:600], width=100))
print('\nURL:',url)
PY


Output:
Exit code: 0
Wall time: 0.2 seconds
Output:
Title: Vistaar: Diverse Benchmarks and Training Sets for Indian Language ASR

Abstract (first 600 chars):
Improving ASR systems is necessary to make new LLM-based use-cases accessible to people across the
globe. In this paper, we focus on Indian languages, and make the case that diverse benchmarks are
required to evaluate and improve ASR systems for Indian languages. To address this, we collate
Vistaar as a set of 59 benchmarks across various language and domain combinations, on which we
evaluate 3 publicly available ASR systems and 2 commercial systems. We also train IndicWhisper
models by fine-tuning the Whisper models on publicly available training datasets across 12 Indian
languages totalling

URL: https://arxiv.org/abs/2305.15386


**Searching for benchmarks**

I'm considering that we could rely on existing reference docs since they contain numbers, but those might be made up by other AI agents. The user requested an internet search, so I want to find reliable sources instead. They specifically asked for benchmarks on datasets and accuracy scores or WER scores for our pipeline, which implies they want to benchmark it. So, I'll search for suitable datasets, like the FLEURS dataset and others, to provide useful suggestions!


**Evaluating datasets for WER**

I'm focusing on the pipeline for offline transcription of single speaker segments. To evaluate minimal WER, I need reference transcripts, so I'll look for datasets that provide audio segments along with their transcripts in various languages. There are several available, like Common Voice by Mozilla, which includes many Indian languages. It offers quality read speech but might differ from the dynamic nature of YouTube podcasts. Another option is the FLEURS dataset from Google, covering 102 languages, including many Indian languages as well.


**Exploring additional datasets**

I'm considering various options for datasets that could be useful. For instance, there’s IndicSUPERB or IndicASR tasks, and MUCS includes Hindi-English code-switching. The Svarah dataset from AI4Bharat focuses on accents. Vistaar is a benchmark suite with multiple datasets, though downloading them may be tricky. AI4Bharat also offers the "Bhashini" dataset, and Google’s FLEURS has potential. OpenSLR includes Indian language corpora like the Hindi speech recognition dataset. I should consider a quick search for specific datasets like the "OpenSLR Gujarati ASR dataset."


**Providing benchmark instructions**

The user wants to know how to perform benchmarks to gather language-wise performance statistics. I can propose implementing a download script using Hugging Face datasets. However, since they asked not to create new files, I can provide step-by-step command-line instructions instead.

First, I’ll suggest installing dependencies with `pip install -e '.[full]'` and `pip install datasets soundfile`. Then, for each dataset, we can sample N segments per language, around 500. Finally, we’ll transcribe using `maya-transcribe run` and produce a `results.jsonl`, while ensuring the `refs.jsonl` matches our segment IDs accurately.


**Computing segment IDs and dataset selection**

To generate `refs.jsonl`, I need to compute `segment_id` by replicating the `segment_identity` logic—using a base_id from the audio file stem and a uid from the SHA1 of the relative path. By storing audio files in a specific structure under `segments_dir`, I can create segment_ids. After running the pipeline, I'll have `results.jsonl` with segment_ids ready to generate `refs.jsonl`.

I'll also evaluate results using `maya-transcribe eval` for WER/CER stats and analyze patterns based on language gaps. For dataset selection, I have options like FLEURS, Common Voice, and possibly AI4Bharat datasets. I’ll search Hugging Face for more details on the Vistaar benchmark suite.


python3 - <<'PY'
import requests, json
url='https://huggingface.co/api/datasets?search=vistaar'
print('GET',url)
resp=requests.get(url,timeout=20)
print('status',resp.status_code)
items=resp.json()
print('count',len(items))
for it in items[:10]:
    print('-', it.get('id'))
PY


Output:
Exit code: 0
Wall time: 0.3 seconds
Output:
GET https://huggingface.co/api/datasets?search=vistaar
status 200
count 4
- iiBLACKii/Gujarati_Vistaar_DB
- ekacare/vistaar_small_asr_eval
- AdityK2409/vistaar_small_asr_eval
- AdityK2409/vistaar_small_asr_eval_2


Summary: auto


python3 - <<'PY'
import requests
url='https://huggingface.co/api/datasets?author=ai4bharat'
resp=requests.get(url,timeout=20)
print('status',resp.status_code)
items=resp.json()
print('datasets',len(items))
for it in items[:30]:
    print('-', it.get('id'))
PY


Output:
Exit code: 0
Wall time: 0.4 seconds
Output:
status 200
datasets 70
- ai4bharat/Aksharantar
- ai4bharat/indicvoices_r
- ai4bharat/Kathbath
- ai4bharat/indic_glue
- ai4bharat/MILU
- ai4bharat/IndicVoices-ST
- ai4bharat/Svarah
- ai4bharat/IndicVoices
- ai4bharat/samanantar
- ai4bharat/IndicParaphrase
- ai4bharat/IndicHeadlineGeneration
- ai4bharat/IndicSentenceSummarization
- ai4bharat/IndicWikiBio
- ai4bharat/IndicQuestionGeneration
- ai4bharat/indicwav2vec_noa_hf
- ai4bharat/IndicQA
- ai4bharat/IndicCOPA
- ai4bharat/IndicXParaphrase
- ai4bharat/IndicSentiment
- ai4bharat/naamapadam
- ai4bharat/Bhasha-Abhijnaanam
- ai4bharat/IN22-Gen
- ai4bharat/IN22-Conv
- ai4bharat/IndicSentiment-Translated
- ai4bharat/IndicXNLI-Translated
- ai4bharat/IndicXParaphrase-Translated
- ai4bharat/IndicCOPA-Translated
- ai4bharat/IndicWikiBio-Translated
- ai4bharat/IndicHeadlineGeneration-Translated
- ai4bharat/ai2_arc-hi


**Exploring datasets for ASR**

I’ve got a list of datasets relevant for Automatic Speech Recognition (ASR): IndicVoices, Svarah, Kathbath, and possibly IndicVoices-ST. It’s exciting to propose these for benchmarking! I need to check if they include audio and transcripts, so I’ll look at the dataset card on Hugging Face. For instance, `ai4bharat/IndicVoices` seems large, and `ai4bharat/Svarah` might focus on accents with transcripts. I need to verify languages like Punjabi and Gujarati; let’s fetch metadata from the Hugging Face API!


python3 - <<'PY'
import requests, json
for ds in ['ai4bharat/IndicVoices','ai4bharat/Svarah','ai4bharat/Kathbath','ai4bharat/IndicVoices-ST','google/fleurs','mozilla-foundation/common_voice_17_0']:
    url=f'https://huggingface.co/api/datasets/{ds}'
    r=requests.get(url,timeout=20)
    print('\n',ds,'status',r.status_code)
    if r.status_code!=200:
        continue
    j=r.json()
    print('downloads:', j.get('downloads'))
    tags=j.get('tags') or []
    # show language tags if present
    lang=[t for t in tags if t.startswith('language:')][:20]
    print('lang tags count',len([t for t in tags if t.startswith('language:')]))
    print('sample lang tags',lang[:10])
    print('task tags sample',[t for t in tags if t.startswith('task')][:10])
    print('cardData keys', list((j.get('cardData') or {}).keys())[:10])
PY


Output:
Exit code: 0
Wall time: 2.7 seconds
Output:

 ai4bharat/IndicVoices status 200
downloads: 6845
lang tags count 0
sample lang tags []
task tags sample []
cardData keys ['license', 'dataset_info', 'configs']

 ai4bharat/Svarah status 200
downloads: 658
lang tags count 0
sample lang tags []
task tags sample []
cardData keys ['dataset_info', 'configs']

 ai4bharat/Kathbath status 200
downloads: 2298
lang tags count 0
sample lang tags []
task tags sample []
cardData keys ['license', 'dataset_info', 'configs']

 ai4bharat/IndicVoices-ST status 200
downloads: 223
lang tags count 13
sample lang tags ['language:as', 'language:bn', 'language:gu', 'language:hi', 'language:kn', 'language:ml', 'language:mr', 'language:ne', 'language:or', 'language:pa']
task tags sample []
cardData keys ['license', 'language', 'pretty_name', 'extra_gated_fields', 'size_categories', 'multilinguality', 'dataset_info', 'configs']

 google/fleurs status 200
downloads: 38919
lang tags count 102
sample lang tags ['language:afr', 'language:amh', 'language:ara', 'language:asm', 'language:ast', 'language:azj', 'language:bel', 'language:ben', 'language:bos', 'language:cat']
task tags sample ['task_categories:automatic-speech-recognition']
cardData keys ['annotations_creators', 'language_creators', 'language', 'license', 'multilinguality', 'size_categories', 'task_categories', 'task_ids', 'pretty_name', 'tags']

 mozilla-foundation/common_voice_17_0 status 200
downloads: 1750
lang tags count 0
sample lang tags []
task tags sample ['task_categories:automatic-speech-recognition']
cardData keys ['task_categories']


python3 - <<'PY'
import requests

ds='ai4bharat/IndicVoices'
j=requests.get(f'https://huggingface.co/api/datasets/{ds}',timeout=20).json()
configs=((j.get('cardData') or {}).get('configs') or [])
print('configs',len(configs))
print('first 30 configs:', [c.get('config_name') for c in configs[:30]])

# Same for Svarah
for ds in ['ai4bharat/Svarah','ai4bharat/Kathbath','ai4bharat/IndicVoices-ST']:
    j=requests.get(f'https://huggingface.co/api/datasets/{ds}',timeout=20).json()
    configs=((j.get('cardData') or {}).get('configs') or [])
    print('\n',ds,'configs',len(configs))
    print('first 30:', [c.get('config_name') for c in configs[:30]])
PY