Output:
Exit code: 0
Wall time: 0 seconds
Output:
                    "relpath": ident.relpath,
                    "uid": ident.uid,
                    "src_path": str(src),
                    "wav_path": str(wav_path),
                    "src_ffprobe": asdict(src_info),
                    "ffprobe": asdict(info),
                    "qc": asdict(qc),
                    "lid": (
                        None
                        if lid is None
                        else {
                            "model": lid.model,
                            "predictions": [{"code": p.code, "prob": p.prob} for p in lid.predictions],
                        }
                    ),
                    "route": {"language_hint": route_lang},
                    "asr": {
                        k: {
                            "model": v.model,
                            "text": v.text,
                            "language": v.language,
                            "language_probability": v.language_probability,
                            "segments": [asdict(s) for s in v.segments],
                            "meta": v.meta,
                        }
                        for k, v in asr_results.items()
                    },
                    "normalized": norm_hyps,
                    "rover": {
                        "text": rover.text,
                        "skeleton_key": rover.skeleton_key,
                        "disagreement_rate": rover.disagreement_rate,
                        "disagreements": [
                            {"kind": d.kind, "index": d.index, "chosen": d.chosen, "options": d.options}
                            for d in rover.disagreements
                        ],
                    },
                    "ctc": ctc_info,
                    "final": {"text": final_text, "source": final_source},
                    "verification": {
                        "decision": verification.decision,
                        "reasons": verification.reasons,
                        "scores": verification.scores,
                    },
                    "config": {
                        "trim_silence": trim_silence,
                        "silence_threshold_db": silence_threshold_db,
                        "min_silence_sec": min_silence_sec,
                        "gemini_model": gemini_model,
                        "gemini_hard_only": gemini_hard_only,
                        "ctc_model": ctc_model,
                    },
                    "timing": {
                        "preprocess_sec": t_pre1 - t_pre0,
                        "lid_sec": t_lid1 - t_lid0,
                        "asr_sec": t_asr1 - t_asr0,
                        "normalize_sec": t_norm1 - t_norm0,
                        "rover_sec": t_rover1 - t_rover0,
                        "ctc_sec": t_ctc1 - t_ctc0,
                        "verify_sec": t_ver1 - t_ver0,
                        "total_sec": time.perf_counter() - t0,
                    },
                }

                f.write(json.dumps(record, ensure_ascii=False) + "\n")
                decision = verification.decision
                counts[decision] = counts.get(decision, 0) + 1
                timings_total.append(record["timing"]["total_sec"])

            except Exception as e:  # pragma: no cover
                counts["error"] += 1
                f.write(
                    json.dumps(
                        {"segment_id": segment_id, "src_path": str(src), "error": str(e)}, ensure_ascii=False
                    )
                    + "\n"
                )

    summary = {
        "segments_total": len(audio_files),
        "counts": counts,
        "avg_total_sec": (sum(timings_total) / len(timings_total)) if timings_total else None,
        "p50_total_sec": _percentile(timings_total, 0.50),
        "p90_total_sec": _percentile(timings_total, 0.90),
    }
    summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    return results_path

def _percentile(xs: list[float], q: float) -> float | None:
    if not xs:
        return None
    xs_sorted = sorted(xs)
    idx = int(round((len(xs_sorted) - 1) * q))
    idx = max(0, min(len(xs_sorted) - 1, idx))
    return float(xs_sorted[idx])


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


nl -ba maya_transcribe/pipeline.py | sed -n '120,260p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   120
   121	    results_path = out_dir / "results.jsonl"
   122	    summary_path = out_dir / "summary.json"
   123
   124	    counts = {"accept": 0, "human": 0, "reject": 0, "error": 0}
   125	    timings_total: list[float] = []
   126
   127	    seen: dict[str, int] = {}
   128	    with results_path.open("w", encoding="utf-8") as f:
   129	        for src in track(audio_files, description="Transcribing"):
   130	            ident = segment_identity(src, root=segments_dir, seen=seen)
   131	            segment_id = ident.segment_id
   132	            t0 = time.perf_counter()
   133
   134	            try:
   135	                # 1) Preprocess
   136	                t_pre0 = time.perf_counter()
   137	                src_info = ffprobe(src)
   138	                wav_path = out_dir / "audio" / f"{segment_id}.wav"
   139	                normalize_audio(
   140	                    src,
   141	                    wav_path,
   142	                    trim_silence=trim_silence,
   143	                    silence_threshold_db=silence_threshold_db,
   144	                    min_silence_sec=min_silence_sec,
   145	                )
   146	                info = ffprobe(wav_path)
   147	                qc = compute_qc(wav_path)
   148	                t_pre1 = time.perf_counter()
   149
   150	                # 2) LID
   151	                t_lid0 = time.perf_counter()
   152	                lid: LIDResult | None = None
   153	                if lid_backend is not None:
   154	                    lid = lid_backend.predict(wav_path, top_k=5)
   155	                t_lid1 = time.perf_counter()
   156
   157	                # Route language hint (only if confident).
   158	                route_lang = None
   159	                if lid and lid.predictions:
   160	                    top1 = lid.predictions[0]
   161	                    top2p = lid.predictions[1].prob if len(lid.predictions) > 1 else 0.0
   162	                    if float(top1.prob) >= float(thresholds.get("lid_min_prob", 0.90)) and (
   163	                        float(top1.prob) - float(top2p)
   164	                    ) >= float(thresholds.get("lid_min_margin", 0.20)):
   165	                        route_lang = top1.code
   166
   167	                # 3) ASR ensemble
   168	                t_asr0 = time.perf_counter()
   169	                asr_results: dict[str, ASRResult] = {}
   170
   171	                # Whisper: deterministic + a small sampling diversity pass.
   172	                det = whisper_backend.transcribe(wav_path, language=route_lang, beam_size=5, best_of=5, temperature=0.0)
   173	                primary_asr_key = f"{det.model}|det"
   174	                asr_results[primary_asr_key] = det
   175	                samp = whisper_backend.transcribe(
   176	                    wav_path, language=route_lang, beam_size=5, best_of=10, temperature=0.2
   177	                )
   178	                asr_results[f"{samp.model}|samp_t0.2"] = samp
   179
   180	                for tf_backend in transformers_backends:
   181	                    res = tf_backend.transcribe(wav_path, language=route_lang, num_beams=5, max_new_tokens=256)
   182	                    asr_results[res.model] = res
   183
   184	                # Gemini: optional extra hypothesis (prefer running only on "hard" segments).
   185	                if gemini_backend is not None:
   186	                    run_gemini = True
   187	                    if gemini_hard_only:
   188	                        det_sum = _asr_summary(det)
   189	                        avg_lp = det_sum.get("avg_logprob_mean")
   190	                        comp = det_sum.get("compression_ratio_mean")
   191	                        no_speech = det_sum.get("no_speech_prob_mean")
   192	                        run_gemini = bool(
   193	                            route_lang is None
   194	                            or (no_speech is not None and float(no_speech) > 0.5)
   195	                            or (avg_lp is not None and float(avg_lp) < float(thresholds.get("min_avg_logprob_mean", -1.0)))
   196	                            or (
   197	                                comp is not None
   198	                                and float(comp) > float(thresholds.get("max_compression_ratio_mean", 2.4))
   199	                            )
   200	                        )
   201
   202	                    if run_gemini:
   203	                        try:
   204	                            g = gemini_backend.transcribe(wav_path, language=route_lang)
   205	                            asr_results[g.model] = g
   206	                        except Exception:
   207	                            pass
   208
   209	                t_asr1 = time.perf_counter()
   210
   211	                # 4) Normalize hypotheses
   212	                t_norm0 = time.perf_counter()
   213	                norm_hyps: dict[str, str] = {}
   214	                for k, res in asr_results.items():
   215	                    lang_hint = res.language or route_lang
   216	                    norm_hyps[k] = normalize_text(res.text, lang=lang_hint)
   217	                t_norm1 = time.perf_counter()
   218
   219	                # 5) ROVER
   220	                t_rover0 = time.perf_counter()
   221	                rover = rover_combine(norm_hyps)
   222	                t_rover1 = time.perf_counter()
   223
   224	                # 5.5) Audio-grounded fit scoring (optional): choose best candidate by CTC loss.
   225	                t_ctc0 = time.perf_counter()
   226	                ctc_info = None
   227	                final_text = rover.text
   228	                final_source = "rover"
   229	                if ctc_scorer is not None and rank_by_ctc is not None:
   230	                    candidates = {"rover": rover.text, **norm_hyps}
   231	                    ctc_info = rank_by_ctc(ctc_scorer, wav_path, candidates=candidates, lang=route_lang)
   232	                    best_key = ctc_info.get("best_key") if isinstance(ctc_info, dict) else None
   233	                    if best_key and best_key in candidates:
   234	                        final_text = candidates[best_key]
   235	                        final_source = "rover" if best_key == "rover" else f"asr:{best_key}"
   236	                t_ctc1 = time.perf_counter()
   237
   238	                # 6) Verify
   239	                t_ver0 = time.perf_counter()
   240	                asr_summaries = {k: _asr_summary(v) for k, v in asr_results.items()}
   241	                verification = verify_segment(
   242	                    qc=asdict(qc),
   243	                    lid=lid,
   244	                    final_text=final_text,
   245	                    rover={
   246	                        "text": rover.text,
   247	                        "disagreement_rate": rover.disagreement_rate,
   248	                        "skeleton_key": rover.skeleton_key,
   249	                    },
   250	                    asr_summaries=asr_summaries,
   251	                    primary_asr_key=primary_asr_key,
   252	                    ctc=ctc_info,
   253	                    thresholds=thresholds,
   254	                )
   255	                t_ver1 = time.perf_counter()
   256
   257	                record = {
   258	                    "segment_id": segment_id,
   259	                    "base_id": ident.base_id,
   260	                    "relpath": ident.relpath,


Summary: auto


sed -n '1,260p' maya_transcribe/verify.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

from dataclasses import dataclass
from typing import Any

from maya_transcribe.lid import LIDResult

@dataclass(frozen=True)
class Verification:
    decision: str  # "accept" | "human" | "reject"
    reasons: list[str]
    scores: dict[str, Any]

def lid_confidence(lid: LIDResult | None) -> dict[str, Any]:
    if not lid or not lid.predictions:
        return {"available": False, "top1": None, "top1_prob": None, "margin": None, "confident": False}
    top1 = lid.predictions[0]
    top2_prob = lid.predictions[1].prob if len(lid.predictions) > 1 else 0.0
    margin = float(top1.prob) - float(top2_prob)
    return {
        "available": True,
        "top1": top1.code,
        "top1_prob": float(top1.prob),
        "margin": margin,
        "confident": False,
    }

def verify_segment(
    *,
    qc: dict[str, Any],
    lid: LIDResult | None,
    final_text: str,
    rover: dict[str, Any],
    asr_summaries: dict[str, Any],
    primary_asr_key: str | None = None,
    ctc: dict[str, Any] | None = None,
    thresholds: dict[str, Any] | None = None,
) -> Verification:
    """
    Conservative verification gate (precision-first).
    """
    thresholds = thresholds or {}
    min_duration = float(thresholds.get("min_duration_sec", 0.8))
    max_clipping = float(thresholds.get("max_clipping_fraction", 0.01))
    max_disagreement = float(thresholds.get("max_rover_disagreement_rate", 0.03))
    lid_min_prob = float(thresholds.get("lid_min_prob", 0.90))
    lid_min_margin = float(thresholds.get("lid_min_margin", 0.20))
    require_lid = bool(thresholds.get("require_lid", True))

    # Whisper-style failure heuristics (see OpenAI whisper fallback logic).
    min_avg_logprob = float(thresholds.get("min_avg_logprob_mean", -1.0))
    max_compression_ratio = float(thresholds.get("max_compression_ratio_mean", 2.4))

    # CTC fit (audio-grounded) gating.
    require_ctc = bool(thresholds.get("require_ctc", False))
    min_ctc_margin = float(thresholds.get("min_ctc_margin", 0.0))

    reasons: list[str] = []
    decision = "accept"

    duration = float(qc.get("duration_sec") or 0.0)
    clipping = float(qc.get("clipping_fraction") or 0.0)
    if duration < min_duration:
        decision = "reject"
        reasons.append("too_short")
    if clipping > max_clipping:
        decision = "human" if decision != "reject" else decision
        reasons.append("clipping")

    lidc = lid_confidence(lid)
    if lidc["available"]:
        lidc["confident"] = bool(lidc["top1_prob"] >= lid_min_prob and lidc["margin"] >= lid_min_margin)
        if not lidc["confident"]:
            decision = "human" if decision != "reject" else decision
            reasons.append("lid_uncertain")
    else:
        reasons.append("lid_missing")
        if require_lid:
            decision = "human" if decision != "reject" else decision

    disagreement_rate = float(rover.get("disagreement_rate") or 0.0)
    if disagreement_rate > max_disagreement:
        decision = "human" if decision != "reject" else decision
        reasons.append("asr_disagreement")

    final_text = (final_text or "").strip()
    if "<unk>" in final_text or "<inaudible>" in final_text:
        decision = "human" if decision != "reject" else decision
        reasons.append("contains_unk")

    # Whisper-style safety: flag if any decoder claims "no speech" with high probability.
    no_speech_probs = []
    for summary in asr_summaries.values():
        p = summary.get("no_speech_prob_mean")
        if p is None:
            continue
        no_speech_probs.append(float(p))
    if no_speech_probs and max(no_speech_probs) > 0.8:
        decision = "reject"
        reasons.append("no_speech")

    # Whisper-style hallucination heuristics: apply to the primary ASR (or rover skeleton fallback).
    asr_key_for_heuristics = primary_asr_key or str(rover.get("skeleton_key") or "")
    heur = asr_summaries.get(asr_key_for_heuristics, {}) if asr_key_for_heuristics else {}
    avg_lp = heur.get("avg_logprob_mean")
    comp = heur.get("compression_ratio_mean")
    if avg_lp is not None and float(avg_lp) < min_avg_logprob:
        decision = "human" if decision != "reject" else decision
        reasons.append("low_logprob")
    if comp is not None and float(comp) > max_compression_ratio:
        decision = "human" if decision != "reject" else decision
        reasons.append("high_compression_ratio")

    # Audio-grounded fit check (when available).
    if require_ctc and (ctc is None or ctc.get("best_loss") is None):
        decision = "human" if decision != "reject" else decision
        reasons.append("ctc_missing")
    if ctc is not None:
        margin = ctc.get("margin")
        if margin is not None and float(margin) < min_ctc_margin:
            decision = "human" if decision != "reject" else decision
            reasons.append("ctc_ambiguous")

    scores = {
        "duration_sec": duration,
        "clipping_fraction": clipping,
        "rover_disagreement_rate": disagreement_rate,
        "lid": lidc,
        "no_speech_prob_max": max(no_speech_probs) if no_speech_probs else None,
        "heuristics_asr_key": asr_key_for_heuristics or None,
        "avg_logprob_mean": avg_lp,
        "compression_ratio_mean": comp,
        "ctc": ctc,
    }

    return Verification(decision=decision, reasons=reasons, scores=scores)


**Considering training data handling**

If we want to train, we might use raw transcripts from selected candidates, ensuring we can map them back to their original sources. Currently, the final source can be "rover" or "asr:somekey," while the final text is normalized. They might need to store this raw final text as well. It’s important to stay mindful of punctuation since the system doesn't remove certain symbols. If output includes "<unk>", it may remain after normalization, impacting the final text.


**Inspecting CTC scoring**

I need to check the `ctc_fit.py` file to verify if the CTC scoring implementation is correct. It’s essential to ensure everything is functioning as intended. I should look into this thoroughly, as any mistake here could affect the scoring process. By examining the code, I'll be able to identify any issues or confirm its accuracy. It’s a good step to ensure quality in the training or evaluation process!


sed -n '1,260p' maya_transcribe/ctc_fit.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Any

from maya_transcribe.audio import decode_pcm16
from maya_transcribe.textnorm import normalize_text

@dataclass(frozen=True)
class CTCScore:
    model_id: str
    loss: float
    text: str

class _CTCModel:
    def __init__(self, model_id: str, device: str | None = None) -> None:
        try:
            import torch
            from transformers import AutoModelForCTC, AutoProcessor
        except Exception as e:  # pragma: no cover
            raise RuntimeError(
                "CTC scoring requires torch + transformers. Install with: pip install -e '.[torch]'"
            ) from e

        if device is None:
            device = "cuda" if torch.cuda.is_available() else "cpu"

        self.model_id = model_id
        self.device = device
        self._torch = torch
        self._processor = AutoProcessor.from_pretrained(model_id)
        self._model = AutoModelForCTC.from_pretrained(model_id).to(device)
        self._model.eval()

        blank = getattr(self._model.config, "pad_token_id", None)
        if blank is None and getattr(self._processor, "tokenizer", None) is not None:
            blank = getattr(self._processor.tokenizer, "pad_token_id", None)
        self.blank_id = int(blank) if blank is not None else 0

    def score(self, audio_path: Path, text: str, lang: str | None = None) -> float:
        import torch.nn.functional as F

        if not text:
            raise ValueError("empty_text")
        if "<unk>" in text or "<inaudible>" in text:
            raise ValueError("contains_unk")

        wav, sr = decode_pcm16(audio_path, sample_rate=16000, mono=True)
        if wav.size == 0:
            raise ValueError("empty_audio")

        # Conservative normalization helps tokenizer compatibility.
        text_norm = normalize_text(text, lang=lang)
        if not text_norm:
            raise ValueError("empty_text")

        inputs = self._processor(wav, sampling_rate=sr, return_tensors="pt")

        # Common key names: input_values (wav2vec2) or input_features (some processors).
        if "input_values" in inputs:
            model_inputs = {"input_values": inputs["input_values"].to(self.device)}
        elif "input_features" in inputs:
            model_inputs = {"input_features": inputs["input_features"].to(self.device)}
        else:
            # Best-effort: move all tensors to device.
            model_inputs = {k: v.to(self.device) for k, v in inputs.items() if hasattr(v, "to")}

        with self._torch.no_grad():
            logits = self._model(**model_inputs).logits  # (B,T,C)
            log_probs = logits.log_softmax(dim=-1)  # (B,T,C)

        # Tokenize targets.
        tokenizer = getattr(self._processor, "tokenizer", None)
        if tokenizer is None:
            raise ValueError("missing_tokenizer")
        target_ids = tokenizer(text_norm).get("input_ids")
        if not target_ids:
            raise ValueError("empty_targets")

        # torch ctc_loss expects (T,N,C)
        log_probs_tnc = log_probs.transpose(0, 1)
        device = log_probs_tnc.device
        input_lengths = self._torch.tensor([log_probs_tnc.size(0)], dtype=self._torch.long, device=device)
        targets = self._torch.tensor(target_ids, dtype=self._torch.long, device=device)
        target_lengths = self._torch.tensor([targets.numel()], dtype=self._torch.long, device=device)

        # Reduction=mean normalizes by target length; good for comparability.
        loss = F.ctc_loss(
            log_probs_tnc,
            targets,
            input_lengths,
            target_lengths,
            blank=self.blank_id,
            reduction="mean",
            zero_infinity=True,
        )
        return float(loss.detach().cpu().item())

class CTCFitScorer:
    """
    Cache-loaded CTC scoring models for audio-grounded transcript fit.
    """

    def __init__(self, model_map: dict[str, str] | None = None, default_model: str | None = None) -> None:
        self.model_map = model_map or {}
        self.default_model = default_model
        self._cache: dict[str, _CTCModel] = {}

    def _get_model_id(self, lang: str | None) -> str | None:
        if lang and lang in self.model_map:
            return self.model_map[lang]
        return self.default_model

    def score(self, audio_path: Path, text: str, lang: str | None) -> CTCScore | None:
        model_id = self._get_model_id(lang)
        if not model_id:
            return None

        model = self._cache.get(model_id)
        if model is None:
            model = _CTCModel(model_id=model_id)
            self._cache[model_id] = model

        loss = model.score(audio_path=audio_path, text=text, lang=lang)
        return CTCScore(model_id=model_id, loss=loss, text=text)

def rank_candidates_by_ctc(
    scorer: CTCFitScorer, audio_path: Path, candidates: dict[str, str], lang: str | None
) -> dict[str, Any]:
    scored: list[tuple[str, CTCScore]] = []
    errors: dict[str, str] = {}

    for key, text in candidates.items():
        text = (text or "").strip()
        if not text:
            continue
        try:
            s = scorer.score(audio_path=audio_path, text=text, lang=lang)
            if s is None:
                continue
            scored.append((key, s))
        except Exception as e:
            errors[key] = f"{type(e).__name__}:{e}"

    scored.sort(key=lambda kv: kv[1].loss)
    best = scored[0] if scored else None

    # Find the best *different-text* alternative, so duplicates don't create a fake "zero margin".
    second = None
    if best:
        best_text = best[1].text
        for item in scored[1:]:
            if item[1].text != best_text:
                second = item
                break

    margin = (second[1].loss - best[1].loss) if (best and second) else None

    return {
        "model_id": best[1].model_id if best else None,
        "best_key": best[0] if best else None,
        "best_loss": best[1].loss if best else None,
        "second_best_key": second[0] if second else None,
        "second_best_loss": second[1].loss if second else None,
        "margin": float(margin) if margin is not None else None,
        "scores": {k: {"loss": s.loss, "model_id": s.model_id} for k, s in scored},
        "errors": errors,
    }


**Evaluating text normalization and scoring**

I have some concerns here. They normalize text again inside the model’s scoring function, but I think that's okay since it ensures consistency. They also reject text with certain markers before scoring, which makes sense.

However, there's a potential issue with scoring multiple languages. If they only use an English model, it won’t work for other languages like Telugu, leading to scoring errors or high losses. They might need to check for unknown tokens as well and consider implementing a way to handle high unknown token ratios.


**Analyzing audio input processing**

I'm looking at the way they compute audio input features. They depend on the AutoProcessor to return data correctly, which might work for models like HuBERT. However, there's a bigger algorithmic concern. Using raw CTC loss as a fit measure can be tricky because it often increases with transcript length. While mean reduction helps, the longer text still might skew results.

Moreover, scoring needs the same tokenizer as the CTC model. Our normalized transcripts include various scripts, so they’d need a compatible CTC model, especially for languages like those using Devanagari.


**Considering language model mapping**

I noticed that they use the route_lang from MMS Language Identification (LID) as a key for model mapping, which could be something like 'hin' or 'hi.' If LID returns 'hin,' that meshes with the user’s suggestion for the ctc_models.json mapping, so that's good. However, there’s uncertainty if LID might return variations like 'Hindi' or 'hin_Deva.'

Another concern is that candidates passed for CTC scoring may include duplicate text, which they attempt to manage with a margin based on differences in loss scores. The threshold margin is set to 0.05, but I wonder if that might be too small depending on the context.


**Evaluating CTC loss and scoring**

I’m thinking about CTC loss values, which can range from 0.1 to 10. That margin of 0.05 seems small but might work as a default. They also use CTC scoring to select the best candidate for final text, which could help improve Word Error Rate (WER). However, if the CTC model isn’t well-aligned with the language or if candidate texts contain unknown tokens, that might lead to scoring failures.

If no candidates are scored, the final text stays as rover.text. When require_ctc is set to True automatically, segments may be marked as human if the transcripts fail to score due to issues like out-of-vocabulary tokens. While this may prioritize precision, it could be a bit too strict for user expectations. The thresholds are set in the code, so it might take users by surprise if they didn’t expect that behavior.


**Assessing CTC model requirements**

It looks like providing a CTC model indicates that gating is mandatory, which is good for maintaining quality. However, they’ve set the minimum CTC margin to 0.05, which could be an issue since there might only be one candidate for certain languages. In those cases, the margin is None, so verification doesn’t mark it as ambiguous.

As for outputting "final" transcripts, they store the final text and source, which seems reasonable. The final text is normalized, which might be a good step. Also, I see that the verification heuristics apply to the primary ASR key, so that's a positive aspect. Now, I should take a look at the new version of eval.py.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from jiwer import cer, wer

from maya_transcribe.io import read_jsonl
from maya_transcribe.textnorm import normalize_text

@dataclass(frozen=True)
class EvalRow:
    segment_id: str
    language: str | None
    decision: str | None
    duration_sec: float | None
    duration_bucket: str | None
    ref: str
    hyp: str
    wer: float
    cer: float

def load_refs(refs_path: Path) -> dict[str, dict[str, Any]]:
    """
    Read a JSONL file of references.
    Expected keys (any one): `text`, `transcript`, `reference`.
    Required key: `segment_id` (or `id`).
    """
    refs: dict[str, dict[str, Any]] = {}
    for row in read_jsonl(refs_path):
        segment_id = str(row.get("segment_id") or row.get("id") or "").strip()
        if not segment_id:
            continue
        refs[segment_id] = row
    return refs

def _get_text(row: dict[str, Any], keys: tuple[str, ...]) -> str | None:
    for k in keys:
        v = row.get(k)
        if isinstance(v, str) and v.strip():
            return v
    return None

def evaluate(
    results_path: Path,
    refs_path: Path,
    out_path: Path | None = None,
    hypothesis: str = "final",
) -> dict[str, Any]:
    refs = load_refs(refs_path)
    results = read_jsonl(results_path)

    rows: list[EvalRow] = []
    for r in results:
        segment_id = str(r.get("segment_id") or "").strip()
        if not segment_id or segment_id not in refs:
            continue

        ref_row = refs[segment_id]
        ref_text = _get_text(ref_row, ("text", "transcript", "reference"))
        if ref_text is None:
            continue

        hyp_text = extract_hypothesis_text(r, hypothesis=hypothesis)
        if hyp_text is None:
            continue

        lang = None
        route = r.get("route") or {}
        if isinstance(route, dict):
            lang = route.get("language_hint")

        decision = None
        ver = r.get("verification") or {}
        if isinstance(ver, dict):
            decision = ver.get("decision")

        duration_sec = None
        qc = r.get("qc") or {}
        if isinstance(qc, dict):
            try:
                duration_sec = float(qc.get("duration_sec")) if qc.get("duration_sec") is not None else None
            except (TypeError, ValueError):
                duration_sec = None
        bucket = duration_bucket(duration_sec)

        ref_norm = normalize_text(ref_text, lang=lang)
        hyp_norm = normalize_text(hyp_text, lang=lang)
        w = float(wer(ref_norm, hyp_norm))
        c = float(cer(ref_norm.replace(" ", ""), hyp_norm.replace(" ", "")))

        rows.append(
            EvalRow(
                segment_id=segment_id,
                language=lang,
                decision=str(decision) if decision is not None else None,
                duration_sec=duration_sec,
                duration_bucket=bucket,
                ref=ref_text,
                hyp=hyp_text,
                wer=w,
                cer=c,
            )
        )

    by_lang: dict[str, list[float]] = {}
    by_dec: dict[str, list[float]] = {}
    by_bucket: dict[str, list[float]] = {}
    by_lang_cer: dict[str, list[float]] = {}
    by_dec_cer: dict[str, list[float]] = {}
    by_bucket_cer: dict[str, list[float]] = {}
    for row in rows:
        if row.language:
            by_lang.setdefault(row.language, []).append(row.wer)
            by_lang_cer.setdefault(row.language, []).append(row.cer)
        if row.decision:
            by_dec.setdefault(row.decision, []).append(row.wer)
            by_dec_cer.setdefault(row.decision, []).append(row.cer)
        if row.duration_bucket:
            by_bucket.setdefault(row.duration_bucket, []).append(row.wer)
            by_bucket_cer.setdefault(row.duration_bucket, []).append(row.cer)

    summary = {
        "hypothesis": hypothesis,
        "pairs_evaluated": len(rows),
        "wer_mean": (sum(r.wer for r in rows) / len(rows)) if rows else None,
        "cer_mean": (sum(r.cer for r in rows) / len(rows)) if rows else None,
        "wer_p50": _percentile([r.wer for r in rows], 0.50),
        "wer_p90": _percentile([r.wer for r in rows], 0.90),
        "wer_p95": _percentile([r.wer for r in rows], 0.95),
        "cer_p50": _percentile([r.cer for r in rows], 0.50),
        "cer_p90": _percentile([r.cer for r in rows], 0.90),
        "cer_p95": _percentile([r.cer for r in rows], 0.95),
        "segment_error_rate": (sum(1 for r in rows if r.wer > 0.0) / len(rows)) if rows else None,
        "pct_wer_gt_5pct": (sum(1 for r in rows if r.wer > 0.05) / len(rows)) if rows else None,
        "pct_wer_gt_10pct": (sum(1 for r in rows if r.wer > 0.10) / len(rows)) if rows else None,
        "wer_by_language_mean": {k: sum(v) / len(v) for k, v in sorted(by_lang.items())},
        "wer_by_decision_mean": {k: sum(v) / len(v) for k, v in sorted(by_dec.items())},
        "wer_by_duration_bucket_mean": {k: sum(v) / len(v) for k, v in sorted(by_bucket.items())},
        "cer_by_language_mean": {k: sum(v) / len(v) for k, v in sorted(by_lang_cer.items())},
        "cer_by_decision_mean": {k: sum(v) / len(v) for k, v in sorted(by_dec_cer.items())},
        "cer_by_duration_bucket_mean": {k: sum(v) / len(v) for k, v in sorted(by_bucket_cer.items())},
    }

    if out_path is not None:
        out_path.parent.mkdir(parents=True, exist_ok=True)
        out_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")

    return summary

def extract_hypothesis_text(result_row: dict[str, Any], hypothesis: str) -> str | None:
    """
    hypothesis:
      - "final": uses result_row["final"]["text"] (falls back to rover if missing)
      - "rover" (default): uses result_row["rover"]["text"]
      - "asr:<key>": uses result_row["asr"][<key>]["text"]
    """
    hypothesis = (hypothesis or "rover").strip()
    if hypothesis == "final":
        final = result_row.get("final") or {}
        if isinstance(final, dict) and isinstance(final.get("text"), str):
            return final["text"]
        hypothesis = "rover"
    if hypothesis == "rover":
        rover = result_row.get("rover") or {}
        if isinstance(rover, dict) and isinstance(rover.get("text"), str):
            return rover["text"]
        return None

    if hypothesis.startswith("asr:"):
        key = hypothesis[len("asr:") :].strip()
        asr = result_row.get("asr") or {}
        if isinstance(asr, dict):
            entry = asr.get(key) or {}
            if isinstance(entry, dict) and isinstance(entry.get("text"), str):
                return entry["text"]
        return None

    return None

def duration_bucket(duration_sec: float | None) -> str | None:
    if duration_sec is None:
        return None
    if duration_sec < 3.0:
        return "0-3s"
    if duration_sec < 10.0:
        return "3-10s"
    if duration_sec < 30.0:
        return "10-30s"
    if duration_sec < 60.0:
        return "30-60s"
    return "60s+"

def _percentile(xs: list[float], q: float) -> float | None:
    if not xs:
        return None
    xs_sorted = sorted(xs)
    idx = int(round((len(xs_sorted) - 1) * q))
    idx = max(0, min(len(xs_sorted) - 1, idx))
    return float(xs_sorted[idx])


nl -ba maya_transcribe/cli.py | sed -n '1,260p'


Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 260
Output:
     1	from __future__ import annotations
     2
     3	import json
     4	from pathlib import Path
     5
     6	import typer
     7	from dotenv import load_dotenv
     8
     9	from maya_transcribe.audio import compute_qc, ffprobe, normalize_audio
    10	from maya_transcribe.config import Settings
    11	from maya_transcribe.dataset import find_segments_dir
    12	from maya_transcribe.ids import segment_identity
    13	from maya_transcribe.io import write_jsonl
    14	from maya_transcribe.r2 import R2Client
    15	from maya_transcribe.tar import safe_extract_tar
    16
    17	load_dotenv(dotenv_path=Path(".env"), override=False)
    18
    19	app = typer.Typer(add_completion=False, no_args_is_help=True)
    20
    21
    22	@app.command()
    23	def download(
    24	    video_id: str = typer.Option(..., help="YouTube video id (used to locate the tar in R2)."),
    25	    out: Path = typer.Option(Path("data"), help="Output directory."),
    26	    r2_key: str | None = typer.Option(None, help="Exact R2 object key (skips search)."),
    27	    bucket: str | None = typer.Option(None, help="R2 bucket override (defaults to env R2_BUCKET)."),
    28	    prefix: str | None = typer.Option(None, help="Optional R2 prefix override."),
    29	    use_db: bool = typer.Option(False, help="Fetch `r2_tar_key` from Supabase Postgres (DATABASE_URL)."),
    30	    dry_run: bool = typer.Option(False, help="Only show matched key(s); do not download."),
    31	) -> None:
    32	    """
    33	    Download the cleaned/segmented tar for a video from R2 and extract it.
    34	    """
    35	    settings = Settings.load()
    36	    r2 = R2Client.from_settings(settings)
    37	    if bucket:
    38	        r2 = R2Client(
    39	            endpoint_url=r2.endpoint_url,
    40	            access_key_id=r2.access_key_id,
    41	            secret_access_key=r2.secret_access_key,
    42	            bucket=bucket,
    43	        )
    44
    45	    out.mkdir(parents=True, exist_ok=True)
    46	    r2_prefix = prefix if prefix is not None else settings.r2_prefix
    47
    48	    matches: list[str]
    49	    if use_db and not r2_key:
    50	        from maya_transcribe.db import fetch_video_meta
    51
    52	        meta = fetch_video_meta(video_id)
    53	        if meta and meta.r2_tar_key:
    54	            r2_key = meta.r2_tar_key
    55
    56	    if r2_key:
    57	        matches = [r2_key]
    58	    else:
    59	        best = r2.find_video_tar_key(video_id=video_id, prefix=r2_prefix)
    60	        if not best:
    61	            typer.echo("No matching objects found in R2.")
    62	            raise typer.Exit(code=2)
    63	        matches = [best]
    64
    65	    if dry_run:
    66	        for key in matches:
    67	            typer.echo(key)
    68	        return
    69
    70	    key = matches[0]
    71	    tar_path = r2.download_to(key=key, out_dir=out / video_id)
    72	    extract_dir = out / video_id
    73	    safe_extract_tar(tar_path, extract_dir=extract_dir)
    74	    segments_dir = find_segments_dir(extract_dir)
    75	    typer.echo(f"Downloaded: {tar_path}")
    76	    typer.echo(f"Extracted to: {extract_dir}")
    77	    if segments_dir:
    78	        typer.echo(f"Segments dir: {segments_dir}")
    79
    80
    81	@app.command()
    82	def preprocess(
    83	    segments_dir: Path = typer.Option(..., exists=True, file_okay=False, help="Directory containing audio segments."),
    84	    out: Path = typer.Option(Path("outputs/preprocessed"), help="Output directory for normalized audio + QC."),
    85	    sample_rate: int = typer.Option(16000, help="Target sample rate."),
    86	    trim_silence: bool = typer.Option(True, help="Trim leading/trailing silence (ffmpeg silenceremove)."),
    87	    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
    88	    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
    89	) -> None:
    90	    """
    91	    Normalize audio to 16kHz mono WAV, optionally trim silence, and write QC metrics.
    92	    """
    93	    out.mkdir(parents=True, exist_ok=True)
    94
    95	    audio_files: list[Path] = []
    96	    for ext in ("*.wav", "*.mp3", "*.m4a", "*.flac", "*.ogg", "*.opus"):
    97	        audio_files.extend(sorted(segments_dir.rglob(ext)))
    98	    audio_files = sorted(set(audio_files))
    99
   100	    if not audio_files:
   101	        typer.echo(f"No audio files found under: {segments_dir}")
   102	        raise typer.Exit(code=2)
   103
   104	    rows = []
   105	    norm_dir = out / "audio"
   106	    seen: dict[str, int] = {}
   107	    for src in audio_files:
   108	        ident = segment_identity(src, root=segments_dir, seen=seen)
   109	        segment_id = ident.segment_id
   110	        dst = norm_dir / f"{segment_id}.wav"
   111	        normalize_audio(
   112	            src,
   113	            dst,
   114	            sample_rate=sample_rate,
  …2584 chars truncated…on(0.0, help="Temperature."),
   174	) -> None:
   175	    """
   176	    Transcribe a single file with faster-whisper.
   177	    """
   178	    from maya_transcribe.asr import FasterWhisperASR
   179
   180	    asr_backend = FasterWhisperASR(model_size_or_path=model)
   181	    res = asr_backend.transcribe(
   182	        audio_path=audio, language=language, beam_size=beam_size, best_of=best_of, temperature=temperature
   183	    )
   184	    typer.echo(res.text)
   185
   186
   187	@app.command()
   188	def gemini(
   189	    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
   190	    language: str | None = typer.Option(None, help="Language hint (e.g., hin/tam/eng)."),
   191	    model: str = typer.Option("gemini-2.0-flash", help="Gemini model name."),
   192	    prompt_file: Path | None = typer.Option(Path("Prompt.md"), help="Prompt file (defaults to Prompt.md)."),
   193	    max_output_tokens: int = typer.Option(512, help="Max output tokens."),
   194	) -> None:
   195	    """
   196	    Transcribe a single file with Gemini (audio-native LLM). Output is parsed/validated strictly.
   197	    """
   198	    from maya_transcribe.gemini_asr import GeminiASR, GeminiConfig, load_prompt_template
   199
   200	    prompt = load_prompt_template(prompt_file)
   201	    backend = GeminiASR(config=GeminiConfig(model=model, max_output_tokens=max_output_tokens), prompt_template=prompt)
   202	    res = backend.transcribe(audio_path=audio, language=language)
   203	    typer.echo(res.text)
   204
   205
   206	@app.command()
   207	def run(
   208	    segments_dir: Path = typer.Option(..., exists=True, file_okay=False, help="Directory containing audio segments."),
   209	    out: Path = typer.Option(Path("outputs/run"), help="Output directory."),
   210	    whisper_model: str = typer.Option("large-v3", help="Whisper model size/path for faster-whisper."),
   211	    lid_model_id: str = typer.Option("facebook/mms-lid-256", help="HuggingFace MMS-LID model id."),
   212	    require_lid_backend: bool = typer.Option(True, help="Fail if MMS-LID backend can't initialize."),
   213	    transformers_whisper_model: list[str] = typer.Option(
   214	        [], help="Extra whisper-family HF models to run via transformers (repeatable)."
   215	    ),
   216	    gemini_model: str | None = typer.Option(None, help="Optional Gemini model name (adds Gemini as a hypothesis)."),
   217	    gemini_prompt_file: Path | None = typer.Option(Path("Prompt.md"), help="Prompt file for Gemini."),
   218	    gemini_hard_only: bool = typer.Option(True, help="Only call Gemini on 'hard' segments."),
   219	    ctc_model: str | None = typer.Option(None, help="Optional HF CTC model id for audio-grounded fit scoring."),
   220	    ctc_model_map: Path | None = typer.Option(None, help="JSON map {lang_code: hf_model_id} for CTC scoring."),
   221	    require_ctc: bool = typer.Option(True, help="When CTC is enabled, require a CTC score to accept."),
   222	    min_ctc_margin: float = typer.Option(0.05, help="Min (second-best - best) CTC loss margin to auto-accept."),
   223	    trim_silence: bool = typer.Option(True, help="Trim leading/trailing silence (ffmpeg silenceremove)."),
   224	    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
   225	    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
   226	    limit: int | None = typer.Option(None, help="Process only the first N segments."),
   227	) -> None:
   228	    """
   229	    End-to-end pipeline: preprocess -> LID -> ASR ensemble -> ROVER -> verify.
   230	    """
   231	    from maya_transcribe.pipeline import run_pipeline
   232
   233	    ctc_map = None
   234	    if ctc_model_map is not None:
   235	        ctc_map = json.loads(ctc_model_map.read_text(encoding="utf-8"))
   236	        if not isinstance(ctc_map, dict):
   237	            raise typer.BadParameter("ctc_model_map must be a JSON object {lang_code: model_id}")
   238
   239	    thresholds = {}
   240	    if ctc_model or ctc_map:
   241	        thresholds["require_ctc"] = require_ctc
   242	        thresholds["min_ctc_margin"] = float(min_ctc_margin)
   243
   244	    results_path = run_pipeline(
   245	        segments_dir=segments_dir,
   246	        out_dir=out,
   247	        lid_model_id=lid_model_id,
   248	        require_lid_backend=require_lid_backend,
   249	        whisper_model=whisper_model,
   250	        extra_transformers_whisper_models=transformers_whisper_model,
   251	        gemini_model=gemini_model,
   252	        gemini_prompt_path=gemini_prompt_file,
   253	        gemini_hard_only=gemini_hard_only,
   254	        ctc_model=ctc_model,
   255	        ctc_model_map=ctc_map,
   256	        trim_silence=trim_silence,
   257	        silence_threshold_db=silence_threshold_db,
   258	        min_silence_sec=min_silence_sec,
   259	        limit=limit,
   260	        thresholds=thresholds,


nl -ba maya_transcribe/cli.py | sed -n '260,520p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   260	        thresholds=thresholds,
   261	    )
   262	    typer.echo(f"Wrote: {results_path}")
   263	    typer.echo(f"Wrote: {out / 'summary.json'}")
   264
   265
   266	@app.command()
   267	def eval(
   268	    results: Path = typer.Option(..., exists=True, dir_okay=False, help="Pipeline results JSONL (outputs/run/results.jsonl)."),
   269	    refs: Path = typer.Option(..., exists=True, dir_okay=False, help="Reference transcripts JSONL."),
   270	    out: Path | None = typer.Option(None, help="Write summary JSON to this path."),
   271	    hypothesis: str = typer.Option('final', help='Hypothesis to evaluate: "final", "rover", or "asr:<key>".'),
   272	) -> None:
   273	    """
   274	    Compute WER/CER for a hypothesis against a reference JSONL keyed by `segment_id`.
   275	    """
   276	    from maya_transcribe.eval import evaluate
   277
   278	    summary = evaluate(results_path=results, refs_path=refs, out_path=out, hypothesis=hypothesis)
   279	    typer.echo(json.dumps(summary, ensure_ascii=False, indent=2))
   280
   281
   282	if __name__ == "__main__":
   283	    app()


nl -ba maya_transcribe/cli.py | sed -n '80,190p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
    80
    81	@app.command()
    82	def preprocess(
    83	    segments_dir: Path = typer.Option(..., exists=True, file_okay=False, help="Directory containing audio segments."),
    84	    out: Path = typer.Option(Path("outputs/preprocessed"), help="Output directory for normalized audio + QC."),
    85	    sample_rate: int = typer.Option(16000, help="Target sample rate."),
    86	    trim_silence: bool = typer.Option(True, help="Trim leading/trailing silence (ffmpeg silenceremove)."),
    87	    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
    88	    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
    89	) -> None:
    90	    """
    91	    Normalize audio to 16kHz mono WAV, optionally trim silence, and write QC metrics.
    92	    """
    93	    out.mkdir(parents=True, exist_ok=True)
    94
    95	    audio_files: list[Path] = []
    96	    for ext in ("*.wav", "*.mp3", "*.m4a", "*.flac", "*.ogg", "*.opus"):
    97	        audio_files.extend(sorted(segments_dir.rglob(ext)))
    98	    audio_files = sorted(set(audio_files))
    99
   100	    if not audio_files:
   101	        typer.echo(f"No audio files found under: {segments_dir}")
   102	        raise typer.Exit(code=2)
   103
   104	    rows = []
   105	    norm_dir = out / "audio"
   106	    seen: dict[str, int] = {}
   107	    for src in audio_files:
   108	        ident = segment_identity(src, root=segments_dir, seen=seen)
   109	        segment_id = ident.segment_id
   110	        dst = norm_dir / f"{segment_id}.wav"
   111	        normalize_audio(
   112	            src,
   113	            dst,
   114	            sample_rate=sample_rate,
   115	            mono=True,
   116	            trim_silence=trim_silence,
   117	            silence_threshold_db=silence_threshold_db,
   118	            min_silence_sec=min_silence_sec,
   119	        )
   120	        info = ffprobe(dst)
   121	        qc = compute_qc(dst, sample_rate=sample_rate)
   122	        rows.append(
   123	            {
   124	                "segment_id": segment_id,
   125	                "base_id": ident.base_id,
   126	                "relpath": ident.relpath,
   127	                "uid": ident.uid,
   128	                "src_path": str(src),
   129	                "wav_path": str(dst),
   130	                "ffprobe": {
   131	                    "duration_sec": info.duration_sec,
   132	                    "sample_rate": info.sample_rate,
   133	                    "channels": info.channels,
   134	                },
   135	                "qc": {
   136	                    "duration_sec": qc.duration_sec,
   137	                    "rms_dbfs": qc.rms_dbfs,
   138	                    "peak_dbfs": qc.peak_dbfs,
   139	                    "clipping_fraction": qc.clipping_fraction,
   140	                },
   141	            }
   142	        )
   143
   144	    write_jsonl(out / "qc.jsonl", rows)
   145	    typer.echo(f"Wrote: {out / 'qc.jsonl'}")
   146	    typer.echo(f"Normalized audio: {norm_dir}")
   147
   148
   149	@app.command()
   150	def lid(
   151	    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
   152	    model_id: str = typer.Option("facebook/mms-lid-256", help="HuggingFace model id."),
   153	    top_k: int = typer.Option(5, help="Return top-K predictions."),
   154	) -> None:
   155	    """
   156	    Spoken language identification (MMS-LID).
   157	    """
   158	    from maya_transcribe.lid import MMSLID
   159
   160	    model = MMSLID(model_id=model_id)
   161	    res = model.predict(audio_path=audio, top_k=top_k)
   162	    for p in res.predictions:
   163	        typer.echo(f"{p.code}\t{p.prob:.4f}")
   164
   165
   166	@app.command()
   167	def asr(
   168	    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
   169	    language: str | None = typer.Option(None, help="Language hint (ISO-639-1 or ISO-639-3)."),
   170	    model: str = typer.Option("large-v3", help="Whisper model size or path for faster-whisper."),
   171	    beam_size: int = typer.Option(5, help="Beam size."),
   172	    best_of: int = typer.Option(5, help="Best-of (sampling candidates)."),
   173	    temperature: float = typer.Option(0.0, help="Temperature."),
   174	) -> None:
   175	    """
   176	    Transcribe a single file with faster-whisper.
   177	    """
   178	    from maya_transcribe.asr import FasterWhisperASR
   179
   180	    asr_backend = FasterWhisperASR(model_size_or_path=model)
   181	    res = asr_backend.transcribe(
   182	        audio_path=audio, language=language, beam_size=beam_size, best_of=best_of, temperature=temperature
   183	    )
   184	    typer.echo(res.text)
   185
   186
   187	@app.command()
   188	def gemini(
   189	    audio: Path = typer.Option(..., exists=True, dir_okay=False, help="Audio file path."),
   190	    language: str | None = typer.Option(None, help="Language hint (e.g., hin/tam/eng)."),


**Inspecting updates**

I’m looking into the preprocessing that uses segment identity, which seems good. Now the pipeline is using CTC scoring and something called gemini, which is intriguing. I noticed a new file called `textnorm.py`, and it's grown in size from 1050 to 1448. I should definitely inspect this file to see if they've updated the way it handles Latin script for English or any other changes. Let’s check it out!


sed -n '1,220p' maya_transcribe/textnorm.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

import unicodedata

_ZERO_WIDTH = {
    "\u200b",  # zero width space
    "\u200c",  # zero width non-joiner
    "\u200d",  # zero width joiner
    "\ufeff",  # byte order mark
}

def normalize_text(text: str, lang: str | None = None) -> str:
    """
    Conservative normalization for hypothesis alignment/comparison (not for publishing).
    - Unicode NFKC
    - Drop punctuation
    - Normalize all Unicode digits to ASCII
    - Collapse whitespace
    - Lowercase for English only
    """
    if not text:
        return ""

    text = unicodedata.normalize("NFKC", text)

    out_chars: list[str] = []
    for ch in text:
        if ch in _ZERO_WIDTH:
            continue

        cat = unicodedata.category(ch)
        if cat.startswith("P"):
            continue

        if cat == "Zs" or ch in ("\t", "\n", "\r"):
            out_chars.append(" ")
            continue

        if cat == "Nd":
            try:
                out_chars.append(str(unicodedata.digit(ch)))
                continue
            except (TypeError, ValueError):
                pass

        out_chars.append(ch)

    norm = "".join(out_chars)
    norm = " ".join(norm.split())

    if (lang or "").lower() in {"en", "eng"}:
        norm = norm.lower()

    return norm

def tokenize_words(text: str) -> list[str]:
    text = " ".join((text or "").split())
    if not text:
        return []
    return text.split(" ")


**Advising on ASR Models**

I think we should suggest incorporating a “superior model” as an additional hypothesis. For offline computing, it’s best to use specialized ASR models tailored for each language, like IndicWhisper, IndicConformer, or MMS. They asked for offline solutions, which makes using an online API like Gemini less ideal. There could be a preference for offline to save on costs, though there might be some flexibility. For evaluating languages, a combination approach with human correction and code-switching detection could work too.


**Considering Code-Switching in ASR**

I need to think about code-switching and how language identification (LID) works per segment. There might be uncertainty with `route_lang`, especially if segments are code-switched. If `route_lang` is uncertain, Whisper can auto-detect the language, which might be okay. However, for the TransformersWhisperASR, if `route_lang` is None, it defaults to auto-detection. With Gemini, if `route_lang` is None, it prompts "unknown," which isn't very helpful. It might still transcribe, but the output could be inconsistent regarding the script used.


**Improving Language Handling in ASR**

I think it’s better to pass the actual language name instead of including a hint line, which they always do. For uncertain languages, we might avoid running Gemini or pass audio LID as top1, even if uncertain. The Gemini backend returns the ASRResult with language determined by `route_lang` or defaults to "unknown." When normalizing, if `res.language` is "unknown," it won't lowercase, which is fine. But if `route_lang` is None, it might complicate text normalization and scoring, especially for code-switched inputs. It could help to map `route_lang` codes to canonical names and add script info.


**Addressing Audio Normalization Issues**

I see another issue with `normalize_audio` using ffmpeg's `silenceremove` at a -40dB threshold with a 0.15 min duration. This works for clean segments but can cut low-volume speech or breathy starts, negatively affecting WER. Tuning the threshold using measured rms_dbfs might help. They also have a `trim_silence` option to disable that feature. I also noticed that `compute_qc` uses decode_pcm16 via ffmpeg multiple times, which seems okay, but we should handle short segments carefully, especially in ROVER where tokenization might cause issues for Indic languages.


**Proposing Enhancements for ASR Pipeline**

I think we should advise the user to pass the specific model, like `--transformers-whisper-model ai4bharat/indicwhisper`. They treat Gemini as an additional hypothesis and compute normalized hypotheses well, followed by running ROVER. While they set the final source for candidates correctly, it’s important to consider storing both raw and normalized transcripts for training data.

We might also critique their thresholds and propose improving the handling of unknown tokens. Incorporating additional metrics and considering language-specific factors for CTC models could enhance performance significantly. Also, using Gemini as a "challenger" in an ensemble could be beneficial, especially for language verification.


**Assessing Code and Proposing Improvements**

For code-switched segments, I think using specialized models could be beneficial, especially if they seek robustness. A hierarchical approach might work: utilizing local open models initially and escalating to more advanced options, like Gemini Pro, if necessary.

It's essential to handle unknown tokens and transcripts exceeding audio length carefully. I also need to consider the CLI's handling of sensitive data to avoid accidental exposures. Overall, relaxing certain disagreement thresholds could improve accuracy, especially in specific languages. Running static checks for syntax errors and ensuring proper dependencies in the project setup is also vital!


**Analyzing Edge Cases for YouTube Audio**

When considering edge cases for YouTube audio segments, I notice several factors to address, like overlapping speech, music residuals, and very short segments affecting language identification. Code-switching presents unique challenges, especially for podcasts. To manage this complexity, I suggest treating code-switched segments separately to maintain their integrity and normalizing as needed.

We also need to ensure proper evaluation metrics are established, focusing on precision, recall, and consistency in transcripts to improve overall pipeline efficiency. Identifying mis-specified code snippets and adjusting dependencies where necessary will further enhance clarity and functionality.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	This is a prompt for any **audio-native** model (Gemini, Voxtral, etc.) used as an extra ASR hypothesis.
     2	It is written to minimize hallucinations and keep outputs easy to validate.
     3
     4	`prompt`
     5
     6	`Instruction: You are a strict, verbatim transcription engine.
     7	Input: Audio file attached
     8	Language hint (primary): <language>
     9
    10	Task:
    11	- Transcribe only what is clearly spoken in the audio.
    12	- Do NOT guess or “fill in” missing words. If a word or phrase is unclear, output <unk> for that span.
    13	- Preserve repetitions, filler words, stutters, and false starts exactly as spoken.
    14	- Do NOT correct grammar, spelling, or pronunciation.
    15	- Do NOT translate or summarize.
    16	- Do NOT add punctuation.
    17
    18	Code-switching:
    19	- Use the native script for the primary language.
    20	- If English words are spoken, keep them in Latin script as spoken (do not transliterate).
    21
    22	OUTPUT FORMAT (exact):
    23	Output 1: Transcript
    24	<single transcript line>
    25
    26	Do not output anything else (no extra headings, notes, or additional outputs).`


Summary: auto


nl -ba maya_transcribe/gemini_asr.py | sed -n '1,240p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	import os
     4	import time
     5	import unicodedata
     6	from dataclasses import dataclass
     7	from pathlib import Path
     8	from typing import Any
     9
    10	from maya_transcribe.asr import ASRResult
    11	from maya_transcribe.textnorm import normalize_text
    12
    13
    14	def _get_api_key() -> str:
    15	    # Support common env var names; `.env` uses GEMINI_KEY in this repo.
    16	    key = (
    17	        os.getenv("GEMINI_KEY")
    18	        or os.getenv("GEMINI_API_KEY")
    19	        or os.getenv("GOOGLE_API_KEY")
    20	        or os.getenv("GOOGLE_GENAI_API_KEY")
    21	    )
    22	    if not key:
    23	        raise RuntimeError("Missing Gemini API key (set GEMINI_KEY or GOOGLE_API_KEY)")
    24	    return key
    25
    26
    27	def _mime_type_for_path(path: Path) -> str:
    28	    ext = path.suffix.lower().lstrip(".")
    29	    if ext == "wav":
    30	        return "audio/wav"
    31	    if ext == "mp3":
    32	        return "audio/mpeg"
    33	    if ext == "flac":
    34	        return "audio/flac"
    35	    if ext in {"m4a", "mp4"}:
    36	        return "audio/mp4"
    37	    if ext in {"ogg", "opus"}:
    38	        return "audio/ogg"
    39	    return "application/octet-stream"
    40
    41
    42	def load_prompt_template(prompt_path: Path | None = None) -> str:
    43	    """
    44	    Load the prompt text from Prompt.md (preferred) or fall back to a built-in template.
    45	    """
    46	    if prompt_path and prompt_path.exists():
    47	        text = prompt_path.read_text(encoding="utf-8")
    48	        start = text.find("`Instruction:")
    49	        if start != -1:
    50	            start += 1  # drop the leading backtick
    51	            end = text.rfind("`")
    52	            if end != -1 and end > start:
    53	                return text[start:end].strip()
    54	            return text[start:].strip()
    55	        return text.strip()
    56
    57	    return (
    58	        "Instruction: You are a strict, verbatim transcription engine.\n"
    59	        "Input: Audio file attached\n"
    60	        "Language hint (primary): <language>\n\n"
    61	        "Task:\n"
    62	        "- Transcribe only what is clearly spoken in the audio.\n"
    63	        "- Do NOT guess or “fill in” missing words. If unclear, output <unk>.\n"
    64	        "- Preserve repetitions, filler words, stutters, and false starts.\n"
    65	        "- Do NOT correct grammar, spelling, or pronunciation.\n"
    66	        "- Do NOT translate or summarize.\n"
    67	        "- Do NOT add punctuation.\n\n"
    68	        "Code-switching:\n"
    69	        "- Use the native script for the primary language.\n"
    70	        "- If English words are spoken, keep them in Latin script as spoken.\n\n"
    71	        "OUTPUT FORMAT (exact):\n"
    72	        "Output 1: Transcript\n"
    73	        "<single transcript line>\n"
    74	        "Do not output anything else.\n"
    75	    ).strip()
    76
    77
    78	def parse_strict_output(text: str) -> str:
    79	    """
    80	    Extract the transcript line from the strict output format.
    81	    """
    82	    text = (text or "").replace("\r\n", "\n").replace("\r", "\n").strip()
    83	    if not text:
    84	        raise ValueError("empty_response")
    85
    86	    lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
    87	    if not lines:
    88	        raise ValueError("empty_response")
    89
    90	    if lines[0] == "Output 1: Transcript":
    91	        if len(lines) != 2:
    92	            raise ValueError("unexpected_extra_lines")
    93	        transcript = lines[1].strip()
    94	    else:
    95	        # Some model variants return just the transcript; accept only if it's a single non-empty line.
    96	        if len(lines) != 1:
    97	            raise ValueError("unexpected_extra_lines")
    98	        transcript = lines[0].strip()
    99
   100	    if not transcript:
   101	        raise ValueError("empty_transcript")
   102
   103	    # Safety: reject if model added extra outputs.
   104	    lowered = text.lower()
   105	    if "output 2" in lowered or "output 3" in lowered or "output 4" in lowered:
   106	        raise ValueError("extra_outputs")
   107
   108	    # Prompt forbids punctuation; treat punctuation as invalid output.
   109	    for ch in transcript:
   110	        if unicodedata.category(ch).startswith("P"):
   111	            raise ValueError("punctuation_forbidden")
   112
   113	    return transcript
   114
   115
   116	@dataclass(frozen=True)
   117	class GeminiConfig:
   118	    model: str = "gemini-2.0-flash"
   119	    temperature: float = 0.0
   120	    max_output_tokens: int = 512
   121	    response_mime_type: str = "text/plain"
   122	    retries: int = 3
   123	    initial_backoff_sec: float = 1.0
   124
   125
   126	class GeminiASR:
   127	    def __init__(self, config: GeminiConfig | None = None, prompt_template: str | None = None) -> None:
   128	        try:
   129	            from google import genai
   130	            from google.genai import types
   131	        except Exception as e:  # pragma: no cover
   132	            raise RuntimeError("GeminiASR requires `google-genai` (pip install google-genai)") from e
   133
   134	        self._genai = genai
   135	        self._types = types
   136	        self.config = config or GeminiConfig()
   137	        self._client = genai.Client(api_key=_get_api_key())
   138	        self._prompt_template = prompt_template or load_prompt_template(Path("Prompt.md"))
   139
   140	    @property
   141	    def name(self) -> str:
   142	        return f"gemini:{self.config.model}"
   143
   144	    def transcribe(self, audio_path: Path, language: str | None = None) -> ASRResult:
   145	        language = language or "unknown"
   146	        prompt = self._prompt_template.replace("<language>", language)
   147	        audio_bytes = audio_path.read_bytes()
   148	        mime_type = _mime_type_for_path(audio_path)
   149
   150	        cfg = self._types.GenerateContentConfig(
   151	            temperature=self.config.temperature,
   152	            max_output_tokens=self.config.max_output_tokens,
   153	            response_mime_type=self.config.response_mime_type,
   154	        )
   155
   156	        last_err: Exception | None = None
   157	        for attempt in range(self.config.retries + 1):
   158	            try:
   159	                response = self._client.models.generate_content(
   160	                    model=self.config.model,
   161	                    contents=[
   162	                        self._types.Part.from_text(text=prompt),
   163	                        self._types.Part.from_bytes(data=audio_bytes, mime_type=mime_type),
   164	                    ],
   165	                    config=cfg,
   166	                )
   167	                raw_text = getattr(response, "text", None) or ""
   168	                transcript = parse_strict_output(raw_text)
   169
   170	                # Normalization is performed later in the pipeline; keep the raw line.
   171	                return ASRResult(
   172	                    model=self.name,
   173	                    text=transcript,
   174	                    language=language,
   175	                    language_probability=None,
   176	                    segments=[],
   177	                    meta={
   178	                        "mime_type": mime_type,
   179	                        "temperature": self.config.temperature,
   180	                        "max_output_tokens": self.config.max_output_tokens,
   181	                        "response_mime_type": self.config.response_mime_type,
   182	                        "prompt_language": language,
   183	                        "raw_output": raw_text,
   184	                    },
   185	                )
   186	            except Exception as e:
   187	                last_err = e
   188	                if attempt >= self.config.retries:
   189	                    break
   190	                backoff = self.config.initial_backoff_sec * (2**attempt)
   191	                time.sleep(backoff)
   192
   193	        raise RuntimeError(f"gemini_transcribe_failed: {type(last_err).__name__}: {last_err}")
   194
   195
   196	def safe_normalized_transcript(text: str, lang: str | None) -> str:
   197	    """
   198	    Convenience helper: strict normalize for alignment/gating.
   199	    """
   200	    return normalize_text(text, lang=lang)


rg -n "google-genai|genai" pyproject.toml README.md maya_transcribe/cli.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
pyproject.toml:23:gemini = ["google-genai>=1.54.0"]
pyproject.toml:36:  "google-genai>=1.54.0",


**Reviewing pyproject updates**

It looks like the pyproject has been updated with the gemini optional dependency, along with full extra includes for google-genai. It's interesting that our earlier look at the pyproject didn't reveal this change, but now it shows the updated information. I want to be sure to check this latest version to see what other updates might be included, and I'll get on opening the updated pyproject soon!


Output:
Exit code: 0
Wall time: 0 seconds
Output:
[build-system]
requires = ["setuptools>=69", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "maya-transcribe"
version = "0.1.0"
description = "Quality-first offline transcription pipeline (Indic + English)"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
  "boto3>=1.34.0",
  "jiwer>=3.0.4",
  "numpy>=1.24.0",
  "python-dotenv>=1.0.1",
  "rich>=13.7.1",
  "typer>=0.12.3",
]

[project.optional-dependencies]
supabase = ["supabase>=2.6.0"]
db = ["psycopg2-binary>=2.9.9"]
gemini = ["google-genai>=1.54.0"]
asr = [
  "faster-whisper>=1.0.3",
  "soundfile>=0.12.1",
]
torch = [
  "torch>=2.2.0",
  "torchaudio>=2.2.0",
  "transformers>=4.41.0",
]
full = [
  "supabase>=2.6.0",
  "psycopg2-binary>=2.9.9",
  "google-genai>=1.54.0",
  "faster-whisper>=1.0.3",
  "soundfile>=0.12.1",
  "torch>=2.2.0",
  "torchaudio>=2.2.0",
  "transformers>=4.41.0",
]

[project.scripts]
maya-transcribe = "maya_transcribe.cli:app"

[tool.setuptools.packages.find]
where = ["."]
include = ["maya_transcribe*"]


**Checking project updates**

I see that the pyproject has been updated, which is great! Now, I want to check if the README file has also been updated. I'll need to search for any recent changes or updates. It's always good to keep everything in sync, especially when there are updates in the project files. Let's see what I can find in the README!


Output:
Exit code: 0
Wall time: 0 seconds
Output:
# maya-transcribe

Offline, quality-first transcription pipeline for cleaned single-speaker audio segments (Indic + English).

## Install

```bash
python3 -m pip install -U pip
python3 -m pip install -e '.[full]'
```

If you only need R2 download + evaluation (no ASR), install the base deps:

```bash
python3 -m pip install -e .
```

## Configure

Create / update `.env` in the repo root. The CLI auto-loads it.

Minimum for R2:
- `R2_ENDPOINT_URL`
- `R2_ACCESS_KEY_ID`
- `R2_SECRET_ACCESS_KEY`
- `R2_BUCKET` (defaults to `1-cleaned-data` if unset)
- `R2_PREFIX` (optional)

## Usage

```bash
maya-transcribe --help
maya-transcribe download --video-id <VIDEO_ID> --bucket 1-cleaned-data --out data/
maya-transcribe run --segments-dir data/<VIDEO_ID>/**/segments --out outputs/<VIDEO_ID> --whisper-model large-v3
```

Evaluation (requires a reference JSONL keyed by `segment_id`):

```bash
maya-transcribe eval --results outputs/<VIDEO_ID>/results.jsonl --refs refs.jsonl
```

Optional extras:
- Add additional Whisper-family checkpoints: `--transformers-whisper-model <hf_model_id>` (repeatable)
- Add Gemini as an extra hypothesis: `--gemini-model gemini-2.0-flash`
- Enable audio-grounded CTC fit scoring: `--ctc-model <hf_ctc_model_id>` (or `--ctc-model-map ctc_models.json`)


**Analyzing audio processing**

I'm considering the issues that arise with ambiguous audio; the model might select the wrong script. The pipeline has a "route_lang" feature, but if it’s unknown, there could be unpredictable outcomes. It seems better to use a detection prompt like "Please detect language" and still transcribe in the detected language without including detection outputs. Maybe I should skip the gemini call entirely when "route_lang" is None, but the pipeline defaults to calling gemini in such cases, which might complicate things further.


**Examining language identification**

If the language identification (LID) isn't confident, they call gemini with "unknown" as the language prompt, which leads to ambiguity. What is considered the "primary language" in this case? If the model assumes Latin script, it could produce incorrect outputs. The normalization process could be influenced by the language hint, which can be tricky if it's unknown. ROVER might treat gemini’s output as an additional hypothesis. If gemini uses Latin script for Telugu, it risks mismatches during scoring, potentially leading to errors. So, calling gemini could be wasteful without precise language identification.


**Considering gemini and ROVER interactions**

If ROVER uses gemini without CTC, there’s a risk of degradation. If “route_lang” is None and gemini returns Latin, it could further degrade ROVER’s performance. The verification gate might flag high disagreement, but it may still accept it if the disagreement is low. I think we should include gemini hypotheses in ROVER only after normalizing to avoid harm. In uncertain situations, running gemini in detection mode or without script constraints might be better, ensuring safety by specifying a prompt like "Language hint: unknown."


**Addressing output requirements**

They want a strict output line, which is fine. I can suggest, "If the language hint is unknown, detect and transcribe in the correct script." However, we must also keep "Use native script for primary language," even though it risks mis-detection. The strict output rules reject punctuation based on Unicode categories, but some Indic scripts use punctuation-like characters, which might be problematic. Instead of rejecting punctuation entirely, it may be better to strip it, making gemini’s output more usable since it’ll get normalized later anyway.


**Evaluating output handling and improvements**

Rejecting punctuation seems unnecessarily strict and could lower gemini's success rate. To improve word error rate (WER), I think it’s better to allow punctuation and strip it afterward. The prompt suggests no punctuation, but if the model violates that, normalization can salvage it. Instead of raising errors for punctuation, I propose allowing it and letting normalization correct it. Many features in the pipeline, like silence detection, are strong, but evaluation methods could benefit from sophistication, considering metrics like weighted error and audio-text alignment. Users might consider "Gemini-3-pro" for superior modeling in challenging languages.


python3 - <<'PY'
import json
from pathlib import Path
p=Path('outputs/test_run2/results.jsonl')
row=json.loads(p.read_text(encoding='utf-8').splitlines()[0])
print('keys:', sorted(row.keys()))
print('final:', row.get('final'))
print('route:', row.get('route'))
print('verification:', row.get('verification'))
print('ctc:', row.get('ctc'))
print('asr keys:', list((row.get('asr') or {}).keys()))
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
keys: ['asr', 'ffprobe', 'lid', 'normalized', 'qc', 'route', 'rover', 'segment_id', 'src_path', 'timing', 'verification', 'wav_path']
final: None
route: {'language_hint': 'eng'}
verification: {'decision': 'accept', 'reasons': [], 'scores': {'duration_sec': 1.432625, 'clipping_fraction': 0.0, 'rover_disagreement_rate': 0.0, 'lid': {'available': True, 'top1': 'eng', 'top1_prob': 0.999092698097229, 'margin': 0.9989108355948702, 'confident': True}, 'no_speech_prob_max': 0.1619873046875}}
ctc: None
asr keys: ['faster-whisper:tiny|det', 'faster-whisper:tiny|samp_t0.2']


**Evaluating transcript parsing**

I'm thinking about how rejecting punctuation in Gemini parsing might be too strict. Maybe I should allow punctuation and just strip it instead or create a better classification for punctuation marks, since some characters like `’` are common. There's also the issue of handling multi-line transcripts, which ideally should be joined. However, if transcripts wrap or include spontaneous newlines, I still need to process them as single lines. It seems like I should join lines after a label to avoid missing results while keeping in mind the prompt's focus on single-line formats.


**Reviewing processing details**

I need to check for disclaimers, as model outputs sometimes include lines like "Output 1: Transcript." The Gemini API call might have size and timeout issues, so adding streaming could be useful. For larger audio bytes, I might consider using `File` uploading, but it's unnecessary for 60 seconds of audio. Incorporating `safetySettings` can help with content filters during transcription. I should also think about determinism with `seed` and temperature settings. Without additional models, the ensemble lacks diversity, so it's vital to include instructions for the IndicWhisper models.


**Ensuring ID stability**

It's crucial that we ensure ROVER isn't degraded by improper weighting, especially since default weights are equal. I might want to consider giving higher weight to deterministic outputs and incorporating per-key weights. The segment identity uses base_id count to manage collisions, but when running the pipeline, they pass a seen dictionary. Because of different orderings in preprocessing, segment IDs can change, which is an issue. They need to be stable regardless of processing order. Always including uid or combining base_id and uid is essential for reproducibility and evaluation referencing. This needs to be emphasized!


**Addressing segment IDs**

I've realized that segment identity with the seen dictionary uses base_id without uid for the first occurrence, making segment_id dependent on both processing order and subset, which can disrupt the mapping between references and results. For stable IDs, they should be based solely on relative path, rather than run order. A good option would be to consistently use base_id-uid for segment IDs. This way, they ensure stability, which is crucial for datasets with repeated stems across videos, especially when using `--limit` to process segments.


**Preparing review comments**

I need to check if the new code uses `from dotenv import load_dotenv` to load the .env file upon import, which is a good step. Now, I should organize the final response into sections like "What looks good," "Key issues/improvements," "Evaluation suggestions," and "Superior model strategy." While the user expects more detail, I want to keep it moderate and careful, aiming for around 15-20 bullet points without emojis.

For clarity, I’ll provide file references with line numbers formatted properly, like `maya_transcribe/ids.py:12`, and gather appropriate line numbers using `nl -ba`.


nl -ba maya_transcribe/ids.py | sed -n '1,120p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	import hashlib
     4	from dataclasses import dataclass
     5	from pathlib import Path
     6
     7
     8	@dataclass(frozen=True)
     9	class SegmentIdentity:
    10	    segment_id: str
    11	    base_id: str
    12	    relpath: str
    13	    uid: str
    14
    15
    16	def segment_identity(src: Path, root: Path, seen: dict[str, int] | None = None) -> SegmentIdentity:
    17	    """
    18	    Produce a stable segment identity and prevent collisions when stems repeat.
    19	    - `base_id`: `src.stem`
    20	    - `relpath`: path relative to `root` (or absolute as fallback)
    21	    - `uid`: sha1(relpath) prefix
    22	    - `segment_id`: base_id unless it collides; then base_id-uid
    23	    """
    24	    base_id = src.stem
    25	    try:
    26	        relpath = src.relative_to(root).as_posix()
    27	    except Exception:
    28	        relpath = str(src)
    29
    30	    uid = hashlib.sha1(relpath.encode("utf-8")).hexdigest()[:10]
    31	    segment_id = base_id
    32
    33	    if seen is None:
    34	        segment_id = f"{base_id}-{uid}"
    35	    else:
    36	        n = int(seen.get(base_id, 0))
    37	        seen[base_id] = n + 1
    38	        if n > 0:
    39	            segment_id = f"{base_id}-{uid}"
    40
    41	    return SegmentIdentity(segment_id=segment_id, base_id=base_id, relpath=relpath, uid=uid)
    42


Summary: auto


nl -ba maya_transcribe/verify.py | sed -n '1,220p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	from dataclasses import dataclass
     4	from typing import Any
     5
     6	from maya_transcribe.lid import LIDResult
     7
     8
     9	@dataclass(frozen=True)
    10	class Verification:
    11	    decision: str  # "accept" | "human" | "reject"
    12	    reasons: list[str]
    13	    scores: dict[str, Any]
    14
    15
    16	def lid_confidence(lid: LIDResult | None) -> dict[str, Any]:
    17	    if not lid or not lid.predictions:
    18	        return {"available": False, "top1": None, "top1_prob": None, "margin": None, "confident": False}
    19	    top1 = lid.predictions[0]
    20	    top2_prob = lid.predictions[1].prob if len(lid.predictions) > 1 else 0.0
    21	    margin = float(top1.prob) - float(top2_prob)
    22	    return {
    23	        "available": True,
    24	        "top1": top1.code,
    25	        "top1_prob": float(top1.prob),
    26	        "margin": margin,
    27	        "confident": False,
    28	    }
    29
    30
    31	def verify_segment(
    32	    *,
    33	    qc: dict[str, Any],
    34	    lid: LIDResult | None,
    35	    final_text: str,
    36	    rover: dict[str, Any],
    37	    asr_summaries: dict[str, Any],
    38	    primary_asr_key: str | None = None,
    39	    ctc: dict[str, Any] | None = None,
    40	    thresholds: dict[str, Any] | None = None,
    41	) -> Verification:
    42	    """
    43	    Conservative verification gate (precision-first).
    44	    """
    45	    thresholds = thresholds or {}
    46	    min_duration = float(thresholds.get("min_duration_sec", 0.8))
    47	    max_clipping = float(thresholds.get("max_clipping_fraction", 0.01))
    48	    max_disagreement = float(thresholds.get("max_rover_disagreement_rate", 0.03))
    49	    lid_min_prob = float(thresholds.get("lid_min_prob", 0.90))
    50	    lid_min_margin = float(thresholds.get("lid_min_margin", 0.20))
    51	    require_lid = bool(thresholds.get("require_lid", True))
    52
    53	    # Whisper-style failure heuristics (see OpenAI whisper fallback logic).
    54	    min_avg_logprob = float(thresholds.get("min_avg_logprob_mean", -1.0))
    55	    max_compression_ratio = float(thresholds.get("max_compression_ratio_mean", 2.4))
    56
    57	    # CTC fit (audio-grounded) gating.
    58	    require_ctc = bool(thresholds.get("require_ctc", False))
    59	    min_ctc_margin = float(thresholds.get("min_ctc_margin", 0.0))
    60
    61	    reasons: list[str] = []
    62	    decision = "accept"
    63
    64	    duration = float(qc.get("duration_sec") or 0.0)
    65	    clipping = float(qc.get("clipping_fraction") or 0.0)
    66	    if duration < min_duration:
    67	        decision = "reject"
    68	        reasons.append("too_short")
    69	    if clipping > max_clipping:
    70	        decision = "human" if decision != "reject" else decision
    71	        reasons.append("clipping")
    72
    73	    lidc = lid_confidence(lid)
    74	    if lidc["available"]:
    75	        lidc["confident"] = bool(lidc["top1_prob"] >= lid_min_prob and lidc["margin"] >= lid_min_margin)
    76	        if not lidc["confident"]:
    77	            decision = "human" if decision != "reject" else decision
    78	            reasons.append("lid_uncertain")
    79	    else:
    80	        reasons.append("lid_missing")
    81	        if require_lid:
    82	            decision = "human" if decision != "reject" else decision
    83
    84	    disagreement_rate = float(rover.get("disagreement_rate") or 0.0)
    85	    if disagreement_rate > max_disagreement:
    86	        decision = "human" if decision != "reject" else decision
    87	        reasons.append("asr_disagreement")
    88
    89	    final_text = (final_text or "").strip()
    90	    if "<unk>" in final_text or "<inaudible>" in final_text:
    91	        decision = "human" if decision != "reject" else decision
    92	        reasons.append("contains_unk")
    93
    94	    # Whisper-style safety: flag if any decoder claims "no speech" with high probability.
    95	    no_speech_probs = []
    96	    for summary in asr_summaries.values():
    97	        p = summary.get("no_speech_prob_mean")
    98	        if p is None:
    99	            continue
   100	        no_speech_probs.append(float(p))
   101	    if no_speech_probs and max(no_speech_probs) > 0.8:
   102	        decision = "reject"
   103	        reasons.append("no_speech")
   104
   105	    # Whisper-style hallucination heuristics: apply to the primary ASR (or rover skeleton fallback).
   106	    asr_key_for_heuristics = primary_asr_key or str(rover.get("skeleton_key") or "")
   107	    heur = asr_summaries.get(asr_key_for_heuristics, {}) if asr_key_for_heuristics else {}
   108	    avg_lp = heur.get("avg_logprob_mean")
   109	    comp = heur.get("compression_ratio_mean")
   110	    if avg_lp is not None and float(avg_lp) < min_avg_logprob:
   111	        decision = "human" if decision != "reject" else decision
   112	        reasons.append("low_logprob")
   113	    if comp is not None and float(comp) > max_compression_ratio:
   114	        decision = "human" if decision != "reject" else decision
   115	        reasons.append("high_compression_ratio")
   116
   117	    # Audio-grounded fit check (when available).
   118	    if require_ctc and (ctc is None or ctc.get("best_loss") is None):
   119	        decision = "human" if decision != "reject" else decision
   120	        reasons.append("ctc_missing")
   121	    if ctc is not None:
   122	        margin = ctc.get("margin")
   123	        if margin is not None and float(margin) < min_ctc_margin:
   124	            decision = "human" if decision != "reject" else decision
   125	            reasons.append("ctc_ambiguous")
   126
   127	    scores = {
   128	        "duration_sec": duration,
   129	        "clipping_fraction": clipping,
   130	        "rover_disagreement_rate": disagreement_rate,
   131	        "lid": lidc,
   132	        "no_speech_prob_max": max(no_speech_probs) if no_speech_probs else None,
   133	        "heuristics_asr_key": asr_key_for_heuristics or None,
   134	        "avg_logprob_mean": avg_lp,
   135	        "compression_ratio_mean": comp,
   136	        "ctc": ctc,
   137	    }
   138
   139	    return Verification(decision=decision, reasons=reasons, scores=scores)


nl -ba maya_transcribe/ctc_fit.py | sed -n '1,240p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	from dataclasses import dataclass
     4	from pathlib import Path
     5	from typing import Any
     6
     7	from maya_transcribe.audio import decode_pcm16
     8	from maya_transcribe.textnorm import normalize_text
     9
    10
    11	@dataclass(frozen=True)
    12	class CTCScore:
    13	    model_id: str
    14	    loss: float
    15	    text: str
    16
    17
    18	class _CTCModel:
    19	    def __init__(self, model_id: str, device: str | None = None) -> None:
    20	        try:
    21	            import torch
    22	            from transformers import AutoModelForCTC, AutoProcessor
    23	        except Exception as e:  # pragma: no cover
    24	            raise RuntimeError(
    25	                "CTC scoring requires torch + transformers. Install with: pip install -e '.[torch]'"
    26	            ) from e
    27
    28	        if device is None:
    29	            device = "cuda" if torch.cuda.is_available() else "cpu"
    30
    31	        self.model_id = model_id
    32	        self.device = device
    33	        self._torch = torch
    34	        self._processor = AutoProcessor.from_pretrained(model_id)
    35	        self._model = AutoModelForCTC.from_pretrained(model_id).to(device)
    36	        self._model.eval()
    37
    38	        blank = getattr(self._model.config, "pad_token_id", None)
    39	        if blank is None and getattr(self._processor, "tokenizer", None) is not None:
    40	            blank = getattr(self._processor.tokenizer, "pad_token_id", None)
    41	        self.blank_id = int(blank) if blank is not None else 0
    42
    43	    def score(self, audio_path: Path, text: str, lang: str | None = None) -> float:
    44	        import torch.nn.functional as F
    45
    46	        if not text:
    47	            raise ValueError("empty_text")
    48	        if "<unk>" in text or "<inaudible>" in text:
    49	            raise ValueError("contains_unk")
    50
    51	        wav, sr = decode_pcm16(audio_path, sample_rate=16000, mono=True)
    52	        if wav.size == 0:
    53	            raise ValueError("empty_audio")
    54
    55	        # Conservative normalization helps tokenizer compatibility.
    56	        text_norm = normalize_text(text, lang=lang)
    57	        if not text_norm:
    58	            raise ValueError("empty_text")
    59
    60	        inputs = self._processor(wav, sampling_rate=sr, return_tensors="pt")
    61
    62	        # Common key names: input_values (wav2vec2) or input_features (some processors).
    63	        if "input_values" in inputs:
    64	            model_inputs = {"input_values": inputs["input_values"].to(self.device)}
    65	        elif "input_features" in inputs:
    66	            model_inputs = {"input_features": inputs["input_features"].to(self.device)}
    67	        else:
    68	            # Best-effort: move all tensors to device.
    69	            model_inputs = {k: v.to(self.device) for k, v in inputs.items() if hasattr(v, "to")}
    70
    71	        with self._torch.no_grad():
    72	            logits = self._model(**model_inputs).logits  # (B,T,C)
    73	            log_probs = logits.log_softmax(dim=-1)  # (B,T,C)
    74
    75	        # Tokenize targets.
    76	        tokenizer = getattr(self._processor, "tokenizer", None)
    77	        if tokenizer is None:
    78	            raise ValueError("missing_tokenizer")
    79	        target_ids = tokenizer(text_norm).get("input_ids")
    80	        if not target_ids:
    81	            raise ValueError("empty_targets")
    82
    83	        # torch ctc_loss expects (T,N,C)
    84	        log_probs_tnc = log_probs.transpose(0, 1)
    85	        device = log_probs_tnc.device
    86	        input_lengths = self._torch.tensor([log_probs_tnc.size(0)], dtype=self._torch.long, device=device)
    87	        targets = self._torch.tensor(target_ids, dtype=self._torch.long, device=device)
    88	        target_lengths = self._torch.tensor([targets.numel()], dtype=self._torch.long, device=device)
    89
    90	        # Reduction=mean normalizes by target length; good for comparability.
    91	        loss = F.ctc_loss(
    92	            log_probs_tnc,
    93	            targets,
    94	            input_lengths,
    95	            target_lengths,
    96	            blank=self.blank_id,
    97	            reduction="mean",
    98	            zero_infinity=True,
    99	        )
   100	        return float(loss.detach().cpu().item())
   101
   102
   103	class CTCFitScorer:
   104	    """
   105	    Cache-loaded CTC scoring models for audio-grounded transcript fit.
   106	    """
   107
   108	    def __init__(self, model_map: dict[str, str] | None = None, default_model: str | None = None) -> None:
   109	        self.model_map = model_map or {}
   110	        self.default_model = default_model
   111	        self._cache: dict[str, _CTCModel] = {}
   112
   113	    def _get_model_id(self, lang: str | None) -> str | None:
   114	        if lang and lang in self.model_map:
   115	            return self.model_map[lang]
   116	        return self.default_model
   117
   118	    def score(self, audio_path: Path, text: str, lang: str | None) -> CTCScore | None:
   119	        model_id = self._get_model_id(lang)
   120	        if not model_id:
   121	            return None
   122
   123	        model = self._cache.get(model_id)
   124	        if model is None:
   125	            model = _CTCModel(model_id=model_id)
   126	            self._cache[model_id] = model
   127
   128	        loss = model.score(audio_path=audio_path, text=text, lang=lang)
   129	        return CTCScore(model_id=model_id, loss=loss, text=text)
   130
   131
   132	def rank_candidates_by_ctc(
   133	    scorer: CTCFitScorer, audio_path: Path, candidates: dict[str, str], lang: str | None
   134	) -> dict[str, Any]:
   135	    scored: list[tuple[str, CTCScore]] = []
   136	    errors: dict[str, str] = {}
   137
   138	    for key, text in candidates.items():
   139	        text = (text or "").strip()
   140	        if not text:
   141	            continue
   142	        try:
   143	            s = scorer.score(audio_path=audio_path, text=text, lang=lang)
   144	            if s is None:
   145	                continue
   146	            scored.append((key, s))
   147	        except Exception as e:
   148	            errors[key] = f"{type(e).__name__}:{e}"
   149
   150	    scored.sort(key=lambda kv: kv[1].loss)
   151	    best = scored[0] if scored else None
   152
   153	    # Find the best *different-text* alternative, so duplicates don't create a fake "zero margin".
   154	    second = None
   155	    if best:
   156	        best_text = best[1].text
   157	        for item in scored[1:]:
   158	            if item[1].text != best_text:
   159	                second = item
   160	                break
   161
   162	    margin = (second[1].loss - best[1].loss) if (best and second) else None
   163
   164	    return {
   165	        "model_id": best[1].model_id if best else None,
   166	        "best_key": best[0] if best else None,
   167	        "best_loss": best[1].loss if best else None,
   168	        "second_best_key": second[0] if second else None,
   169	        "second_best_loss": second[1].loss if second else None,
   170	        "margin": float(margin) if margin is not None else None,
   171	        "scores": {k: {"loss": s.loss, "model_id": s.model_id} for k, s in scored},
   172	        "errors": errors,
   173	    }


nl -ba maya_transcribe/pipeline.py | sed -n '1,170p'