**Considering data parsing**

I want to ensure that I properly filter the output without skipping necessary lines. If the label isn’t present, I’m thinking of using a list comprehension to get the relevant lines. However, I need to be cautious because "Transcript" could be a regular part of the speech content, so I should only focus on the label format with its colon. I wonder how to handle that effectively without losing important data.


**Considering label processing**

I'm thinking that if there are no labels and multiple lines, I need to join them together. I could include disclaimers, but maybe I can also try to detect and remove lines that contain ":" and are too short. I wonder if that's a bit tricky, though. It feels challenging to get it right and ensure the output is clean and useful! There’s a lot to consider in this process.


Input:
*** Begin Patch
*** Update File: maya_transcribe/gemini_asr.py
@@
-from dataclasses import dataclass
+from dataclasses import dataclass
@@
-from maya_transcribe.asr import ASRResult
+from maya_transcribe.asr import ASRResult
 from maya_transcribe.language import to_prompt_language_hint
 from maya_transcribe.textnorm import normalize_text
@@
 def load_prompt_template(prompt_path: Path | None = None) -> str:
@@
     ).strip()

-
-def parse_strict_output(text: str) -> str:
+@dataclass(frozen=True)
+class ParsedGeminiOutput:
+    transcript: str
+    had_label: bool
+    joined_lines: bool
+    stripped_punctuation: int
+    extra_outputs_present: bool
+
+
+def _strip_punctuation_to_space(text: str) -> tuple[str, int]:
+    out: list[str] = []
+    removed = 0
+    for ch in text:
+        if unicodedata.category(ch).startswith("P"):
+            out.append(" ")
+            removed += 1
+        else:
+            out.append(ch)
+    cleaned = " ".join("".join(out).split())
+    return cleaned, removed
+
+
+def parse_strict_output(text: str) -> ParsedGeminiOutput:
     """
     Extract the transcript line from the strict output format.
     """
     text = (text or "").replace("\r\n", "\n").replace("\r", "\n").strip()
     if not text:
         raise ValueError("empty_response")

-    lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
+    raw_lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
+    # Drop trivial Markdown fences if present.
+    lines = [ln for ln in raw_lines if ln != "```" and not ln.startswith("```")]
     if not lines:
         raise ValueError("empty_response")

-    if lines[0] == "Output 1: Transcript":
-        if len(lines) != 2:
-            raise ValueError("unexpected_extra_lines")
-        transcript = lines[1].strip()
-    else:
-        # Some model variants return just the transcript; accept only if it's a single non-empty line.
-        if len(lines) != 1:
-            raise ValueError("unexpected_extra_lines")
-        transcript = lines[0].strip()
+    lowered = "\n".join(lines).lower()
+    extra_outputs_present = ("output 2" in lowered) or ("output 3" in lowered) or ("output 4" in lowered)
+
+    had_label = False
+    transcript_lines: list[str] = []
+
+    # Prefer the labelled format. Salvage multi-line transcripts by joining.
+    label_idx = None
+    for i, ln in enumerate(lines):
+        if ln.lower().replace(" ", "") == "output1:transcript":
+            label_idx = i
+            break
+
+    if label_idx is not None:
+        had_label = True
+        for ln in lines[label_idx + 1 :]:
+            if ln.lower().startswith("output "):
+                break
+            transcript_lines.append(ln)
+    else:
+        # Unlabelled response: use all lines up to the next "Output ..." section (if any).
+        for ln in lines:
+            if ln.lower().startswith("output "):
+                break
+            transcript_lines.append(ln)
+
+    if not transcript_lines:
+        raise ValueError("empty_transcript")
+
+    joined_lines = len(transcript_lines) > 1
+    transcript = " ".join(transcript_lines).strip()
     if not transcript:
         raise ValueError("empty_transcript")

-    # Safety: reject if model added extra outputs.
-    lowered = text.lower()
-    if "output 2" in lowered or "output 3" in lowered or "output 4" in lowered:
-        raise ValueError("extra_outputs")
-
-    # Prompt forbids punctuation; treat punctuation as invalid output.
-    for ch in transcript:
-        if unicodedata.category(ch).startswith("P"):
-            raise ValueError("punctuation_forbidden")
-
-    return transcript
+    transcript, stripped = _strip_punctuation_to_space(transcript)
+    if not transcript:
+        raise ValueError("empty_transcript")
+
+    return ParsedGeminiOutput(
+        transcript=transcript,
+        had_label=had_label,
+        joined_lines=joined_lines,
+        stripped_punctuation=stripped,
+        extra_outputs_present=extra_outputs_present,
+    )
@@
                 response = self._client.models.generate_content(
                     model=self.config.model,
                     contents=[
                         self._types.Part.from_text(text=prompt),
                         self._types.Part.from_bytes(data=audio_bytes, mime_type=mime_type),
                     ],
                     config=cfg,
                 )
                 raw_text = getattr(response, "text", None) or ""
-                transcript = parse_strict_output(raw_text)
+                parsed = parse_strict_output(raw_text)
+                transcript = parsed.transcript

                 # Normalization is performed later in the pipeline; keep the raw line.
                 return ASRResult(
                     model=self.name,
                     text=transcript,
@@
                         "response_mime_type": self.config.response_mime_type,
                         "prompt_language_hint": prompt_language,
                         "language_code": language_code,
                         "raw_output": raw_text,
+                        "parse": {
+                            "had_label": parsed.had_label,
+                            "joined_lines": parsed.joined_lines,
+                            "stripped_punctuation": parsed.stripped_punctuation,
+                            "extra_outputs_present": parsed.extra_outputs_present,
+                        },
                     },
                 )
*** End Patch


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/maya-transcribe/maya_transcribe/gemini_asr.py:
    ).strip()

def parse_strict_output(text: str) -> str:
    """
    Extract the transcript line from the strict output format.
    """
    text = (text or "").replace("\r\n", "\n").replace("\r", "\n").strip()
    if not text:
        raise ValueError("empty_response")

    lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
    if not lines:
        raise ValueError("empty_response")

    if lines[0] == "Output 1: Transcript":
        if len(lines) != 2:
            raise ValueError("unexpected_extra_lines")
        transcript = lines[1].strip()
    else:
        # Some model variants return just the transcript; accept only if it's a single non-empty line.
        if len(lines) != 1:
            raise ValueError("unexpected_extra_lines")
        transcript = lines[0].strip()
    if not transcript:
        raise ValueError("empty_transcript")

    # Safety: reject if model added extra outputs.
    lowered = text.lower()
    if "output 2" in lowered or "output 3" in lowered or "output 4" in lowered:
        raise ValueError("extra_outputs")

    # Prompt forbids punctuation; treat punctuation as invalid output.
    for ch in transcript:
        if unicodedata.category(ch).startswith("P"):
            raise ValueError("punctuation_forbidden")

    return transcript


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Fixing patch issues**

I'm facing issues with a patch that failed due to a context mismatch because the file changed earlier. To resolve this, I need to re-open `gemini_asr.py` to ensure I'm working with the current content before attempting to patch it again. I'll focus on viewing the relevant portion so I can address the issue correctly. It's all about getting everything aligned!


sed -n '1,260p' maya_transcribe/gemini_asr.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

import os
import time
import unicodedata
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from maya_transcribe.asr import ASRResult
from maya_transcribe.language import to_prompt_language_hint
from maya_transcribe.textnorm import normalize_text

def _get_api_key() -> str:
    # Support common env var names; `.env` uses GEMINI_KEY in this repo.
    key = (
        os.getenv("GEMINI_KEY")
        or os.getenv("GEMINI_API_KEY")
        or os.getenv("GOOGLE_API_KEY")
        or os.getenv("GOOGLE_GENAI_API_KEY")
    )
    if not key:
        raise RuntimeError("Missing Gemini API key (set GEMINI_KEY or GOOGLE_API_KEY)")
    return key

def _mime_type_for_path(path: Path) -> str:
    ext = path.suffix.lower().lstrip(".")
    if ext == "wav":
        return "audio/wav"
    if ext == "mp3":
        return "audio/mpeg"
    if ext == "flac":
        return "audio/flac"
    if ext in {"m4a", "mp4"}:
        return "audio/mp4"
    if ext in {"ogg", "opus"}:
        return "audio/ogg"
    return "application/octet-stream"

def load_prompt_template(prompt_path: Path | None = None) -> str:
    """
    Load the prompt text from Prompt.md (preferred) or fall back to a built-in template.
    """
    if prompt_path and prompt_path.exists():
        text = prompt_path.read_text(encoding="utf-8")
        start = text.find("`Instruction:")
        if start != -1:
            start += 1  # drop the leading backtick
            end = text.rfind("`")
            if end != -1 and end > start:
                return text[start:end].strip()
            return text[start:].strip()
        return text.strip()

    return (
        "Instruction: You are a strict, verbatim transcription engine.\n"
        "Input: Audio file attached\n"
        "Language hint (primary): <language>\n\n"
        "Task:\n"
        "- Transcribe only what is clearly spoken in the audio.\n"
        "- Do NOT guess or “fill in” missing words. If unclear, output <unk>.\n"
        "- Preserve repetitions, filler words, stutters, and false starts.\n"
        "- Do NOT correct grammar, spelling, or pronunciation.\n"
        "- Do NOT translate or summarize.\n"
        "- Do NOT add punctuation.\n\n"
        "Code-switching:\n"
        "- Use the native script for the primary language.\n"
        "- If English words are spoken, keep them in Latin script as spoken.\n\n"
        "OUTPUT FORMAT (exact):\n"
        "Output 1: Transcript\n"
        "<single transcript line>\n"
        "Do not output anything else.\n"
    ).strip()

def parse_strict_output(text: str) -> str:
    """
    Extract the transcript line from the strict output format.
    """
    text = (text or "").replace("\r\n", "\n").replace("\r", "\n").strip()
    if not text:
        raise ValueError("empty_response")

    lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
    if not lines:
        raise ValueError("empty_response")

    if lines[0] == "Output 1: Transcript":
        if len(lines) != 2:
            raise ValueError("unexpected_extra_lines")
        transcript = lines[1].strip()
    else:
        # Some model variants return just the transcript; accept only if it's a single non-empty line.
        if len(lines) != 1:
            raise ValueError("unexpected_extra_lines")
        transcript = lines[0].strip()

    if not transcript:
        raise ValueError("empty_transcript")

    # Safety: reject if model added extra outputs.
    lowered = text.lower()
    if "output 2" in lowered or "output 3" in lowered or "output 4" in lowered:
        raise ValueError("extra_outputs")

    # Prompt forbids punctuation; treat punctuation as invalid output.
    for ch in transcript:
        if unicodedata.category(ch).startswith("P"):
            raise ValueError("punctuation_forbidden")

    return transcript

@dataclass(frozen=True)
class GeminiConfig:
    model: str = "gemini-2.0-flash"
    temperature: float = 0.0
    max_output_tokens: int = 512
    response_mime_type: str = "text/plain"
    retries: int = 3
    initial_backoff_sec: float = 1.0

class GeminiASR:
    def __init__(self, config: GeminiConfig | None = None, prompt_template: str | None = None) -> None:
        try:
            from google import genai
            from google.genai import types
        except Exception as e:  # pragma: no cover
            raise RuntimeError("GeminiASR requires `google-genai` (pip install google-genai)") from e

        self._genai = genai
        self._types = types
        self.config = config or GeminiConfig()
        self._client = genai.Client(api_key=_get_api_key())
        self._prompt_template = prompt_template or load_prompt_template(Path("Prompt.md"))

    @property
    def name(self) -> str:
        return f"gemini:{self.config.model}"

    def transcribe(self, audio_path: Path, language: str | None = None) -> ASRResult:
        language_code = language
        prompt_language = to_prompt_language_hint(language_code)
        prompt = self._prompt_template.replace("<language>", prompt_language)
        audio_bytes = audio_path.read_bytes()
        mime_type = _mime_type_for_path(audio_path)

        cfg = self._types.GenerateContentConfig(
            temperature=self.config.temperature,
            max_output_tokens=self.config.max_output_tokens,
            response_mime_type=self.config.response_mime_type,
        )

        last_err: Exception | None = None
        for attempt in range(self.config.retries + 1):
            try:
                response = self._client.models.generate_content(
                    model=self.config.model,
                    contents=[
                        self._types.Part.from_text(text=prompt),
                        self._types.Part.from_bytes(data=audio_bytes, mime_type=mime_type),
                    ],
                    config=cfg,
                )
                raw_text = getattr(response, "text", None) or ""
                transcript = parse_strict_output(raw_text)

                # Normalization is performed later in the pipeline; keep the raw line.
                return ASRResult(
                    model=self.name,
                    text=transcript,
                    language=language_code,
                    language_probability=None,
                    segments=[],
                    meta={
                        "mime_type": mime_type,
                        "temperature": self.config.temperature,
                        "max_output_tokens": self.config.max_output_tokens,
                        "response_mime_type": self.config.response_mime_type,
                        "prompt_language_hint": prompt_language,
                        "language_code": language_code,
                        "raw_output": raw_text,
                    },
                )
            except Exception as e:
                last_err = e
                if attempt >= self.config.retries:
                    break
                backoff = self.config.initial_backoff_sec * (2**attempt)
                time.sleep(backoff)

        raise RuntimeError(f"gemini_transcribe_failed: {type(last_err).__name__}: {last_err}")

def safe_normalized_transcript(text: str, lang: str | None) -> str:
    """
    Convenience helper: strict normalize for alignment/gating.
    """
    return normalize_text(text, lang=lang)


Summary: auto


Input:
*** Begin Patch
*** Update File: maya_transcribe/gemini_asr.py
@@
 def load_prompt_template(prompt_path: Path | None = None) -> str:
@@
     ).strip()

-
-def parse_strict_output(text: str) -> str:
+@dataclass(frozen=True)
+class ParsedGeminiOutput:
+    transcript: str
+    had_label: bool
+    joined_lines: bool
+    stripped_punctuation: int
+    extra_outputs_present: bool
+
+
+def _strip_punctuation_to_space(text: str) -> tuple[str, int]:
+    out: list[str] = []
+    removed = 0
+    for ch in text:
+        if unicodedata.category(ch).startswith("P"):
+            out.append(" ")
+            removed += 1
+        else:
+            out.append(ch)
+    cleaned = " ".join("".join(out).split())
+    return cleaned, removed
+
+
+def parse_strict_output(text: str) -> ParsedGeminiOutput:
     """
-    Extract the transcript line from the strict output format.
+    Extract the transcript from the strict output format.
+
+    Production note: even with strict prompts, some models may add punctuation or line breaks.
+    We salvage by joining lines and stripping punctuation (since the pipeline normalizes later anyway).
     """
     text = (text or "").replace("\r\n", "\n").replace("\r", "\n").strip()
     if not text:
         raise ValueError("empty_response")

-    lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
+    raw_lines = [ln.strip() for ln in text.split("\n") if ln.strip()]
+    lines = [ln for ln in raw_lines if ln != "```" and not ln.startswith("```")]
     if not lines:
         raise ValueError("empty_response")

-    if lines[0] == "Output 1: Transcript":
-        if len(lines) != 2:
-            raise ValueError("unexpected_extra_lines")
-        transcript = lines[1].strip()
-    else:
-        # Some model variants return just the transcript; accept only if it's a single non-empty line.
-        if len(lines) != 1:
-            raise ValueError("unexpected_extra_lines")
-        transcript = lines[0].strip()
+    lowered = "\n".join(lines).lower()
+    extra_outputs_present = ("output 2" in lowered) or ("output 3" in lowered) or ("output 4" in lowered)

-    if not transcript:
-        raise ValueError("empty_transcript")
+    had_label = False
+    transcript_lines: list[str] = []

-    # Safety: reject if model added extra outputs.
-    lowered = text.lower()
-    if "output 2" in lowered or "output 3" in lowered or "output 4" in lowered:
-        raise ValueError("extra_outputs")
+    def is_output_label(line: str) -> bool:
+        return line.lower().startswith("output ")

-    # Prompt forbids punctuation; treat punctuation as invalid output.
-    for ch in transcript:
-        if unicodedata.category(ch).startswith("P"):
-            raise ValueError("punctuation_forbidden")
+    label_idx = None
+    for i, ln in enumerate(lines):
+        if ln.lower().replace(" ", "") == "output1:transcript":
+            label_idx = i
+            break
+
+    if label_idx is not None:
+        had_label = True
+        for ln in lines[label_idx + 1 :]:
+            if is_output_label(ln):
+                break
+            transcript_lines.append(ln)
+    else:
+        for ln in lines:
+            if is_output_label(ln):
+                break
+            transcript_lines.append(ln)

-    return transcript
+    if not transcript_lines:
+        raise ValueError("empty_transcript")
+
+    joined_lines = len(transcript_lines) > 1
+    transcript = " ".join(transcript_lines).strip()
+    transcript, stripped = _strip_punctuation_to_space(transcript)
+
+    if not transcript:
+        raise ValueError("empty_transcript")
+
+    return ParsedGeminiOutput(
+        transcript=transcript,
+        had_label=had_label,
+        joined_lines=joined_lines,
+        stripped_punctuation=stripped,
+        extra_outputs_present=extra_outputs_present,
+    )
@@
                 )
                 raw_text = getattr(response, "text", None) or ""
-                transcript = parse_strict_output(raw_text)
+                parsed = parse_strict_output(raw_text)
+                transcript = parsed.transcript

                 # Normalization is performed later in the pipeline; keep the raw line.
                 return ASRResult(
                     model=self.name,
                     text=transcript,
@@
                         "response_mime_type": self.config.response_mime_type,
                         "prompt_language_hint": prompt_language,
                         "language_code": language_code,
                         "raw_output": raw_text,
+                        "parse": {
+                            "had_label": parsed.had_label,
+                            "joined_lines": parsed.joined_lines,
+                            "stripped_punctuation": parsed.stripped_punctuation,
+                            "extra_outputs_present": parsed.extra_outputs_present,
+                        },
                     },
                 )
*** End Patch


Output: {"output":"Success. Updated the following files:\nM maya_transcribe/gemini_asr.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


python3 -m compileall -q maya_transcribe


Output:
Exit code: 0
Wall time: 0 seconds
Output:


**Updating pipeline components**

I'm thinking about updating the CLI gemini command since it calls backend.transcribe and prints text, but that part seems unaffected. Next, I should look into updating the pipeline for stable IDs—though I believe I've already taken care of that. Now, I'm concerned about CTC scoring guardrails and coverage metrics, so it seems like I need to check the current ctc_fit.py file for details. It feels a bit overwhelming, but I'm on it!


sed -n '1,260p' maya_transcribe/ctc_fit.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Any

from maya_transcribe.audio import decode_pcm16
from maya_transcribe.textnorm import normalize_text

@dataclass(frozen=True)
class CTCScore:
    model_id: str
    loss: float
    text: str

class _CTCModel:
    def __init__(self, model_id: str, device: str | None = None) -> None:
        try:
            import torch
            from transformers import AutoModelForCTC, AutoProcessor
        except Exception as e:  # pragma: no cover
            raise RuntimeError(
                "CTC scoring requires torch + transformers. Install with: pip install -e '.[torch]'"
            ) from e

        if device is None:
            device = "cuda" if torch.cuda.is_available() else "cpu"

        self.model_id = model_id
        self.device = device
        self._torch = torch
        self._processor = AutoProcessor.from_pretrained(model_id)
        self._model = AutoModelForCTC.from_pretrained(model_id).to(device)
        self._model.eval()

        blank = getattr(self._model.config, "pad_token_id", None)
        if blank is None and getattr(self._processor, "tokenizer", None) is not None:
            blank = getattr(self._processor.tokenizer, "pad_token_id", None)
        self.blank_id = int(blank) if blank is not None else 0

    def score(self, audio_path: Path, text: str, lang: str | None = None) -> float:
        import torch.nn.functional as F

        if not text:
            raise ValueError("empty_text")
        if "<unk>" in text or "<inaudible>" in text:
            raise ValueError("contains_unk")

        wav, sr = decode_pcm16(audio_path, sample_rate=16000, mono=True)
        if wav.size == 0:
            raise ValueError("empty_audio")

        # Conservative normalization helps tokenizer compatibility.
        text_norm = normalize_text(text, lang=lang)
        if not text_norm:
            raise ValueError("empty_text")

        inputs = self._processor(wav, sampling_rate=sr, return_tensors="pt")

        # Common key names: input_values (wav2vec2) or input_features (some processors).
        if "input_values" in inputs:
            model_inputs = {"input_values": inputs["input_values"].to(self.device)}
        elif "input_features" in inputs:
            model_inputs = {"input_features": inputs["input_features"].to(self.device)}
        else:
            # Best-effort: move all tensors to device.
            model_inputs = {k: v.to(self.device) for k, v in inputs.items() if hasattr(v, "to")}

        with self._torch.no_grad():
            logits = self._model(**model_inputs).logits  # (B,T,C)
            log_probs = logits.log_softmax(dim=-1)  # (B,T,C)

        # Tokenize targets.
        tokenizer = getattr(self._processor, "tokenizer", None)
        if tokenizer is None:
            raise ValueError("missing_tokenizer")
        target_ids = tokenizer(text_norm).get("input_ids")
        if not target_ids:
            raise ValueError("empty_targets")

        # torch ctc_loss expects (T,N,C)
        log_probs_tnc = log_probs.transpose(0, 1)
        device = log_probs_tnc.device
        input_lengths = self._torch.tensor([log_probs_tnc.size(0)], dtype=self._torch.long, device=device)
        targets = self._torch.tensor(target_ids, dtype=self._torch.long, device=device)
        target_lengths = self._torch.tensor([targets.numel()], dtype=self._torch.long, device=device)

        # Reduction=mean normalizes by target length; good for comparability.
        loss = F.ctc_loss(
            log_probs_tnc,
            targets,
            input_lengths,
            target_lengths,
            blank=self.blank_id,
            reduction="mean",
            zero_infinity=True,
        )
        return float(loss.detach().cpu().item())

class CTCFitScorer:
    """
    Cache-loaded CTC scoring models for audio-grounded transcript fit.
    """

    def __init__(self, model_map: dict[str, str] | None = None, default_model: str | None = None) -> None:
        self.model_map = model_map or {}
        self.default_model = default_model
        self._cache: dict[str, _CTCModel] = {}

    def _get_model_id(self, lang: str | None) -> str | None:
        if lang and lang in self.model_map:
            return self.model_map[lang]
        return self.default_model

    def score(self, audio_path: Path, text: str, lang: str | None) -> CTCScore | None:
        model_id = self._get_model_id(lang)
        if not model_id:
            return None

        model = self._cache.get(model_id)
        if model is None:
            model = _CTCModel(model_id=model_id)
            self._cache[model_id] = model

        loss = model.score(audio_path=audio_path, text=text, lang=lang)
        return CTCScore(model_id=model_id, loss=loss, text=text)

def rank_candidates_by_ctc(
    scorer: CTCFitScorer, audio_path: Path, candidates: dict[str, str], lang: str | None
) -> dict[str, Any]:
    scored: list[tuple[str, CTCScore]] = []
    errors: dict[str, str] = {}

    for key, text in candidates.items():
        text = (text or "").strip()
        if not text:
            continue
        try:
            s = scorer.score(audio_path=audio_path, text=text, lang=lang)
            if s is None:
                continue
            scored.append((key, s))
        except Exception as e:
            errors[key] = f"{type(e).__name__}:{e}"

    scored.sort(key=lambda kv: kv[1].loss)
    best = scored[0] if scored else None

    # Find the best *different-text* alternative, so duplicates don't create a fake "zero margin".
    second = None
    if best:
        best_text = best[1].text
        for item in scored[1:]:
            if item[1].text != best_text:
                second = item
                break

    margin = (second[1].loss - best[1].loss) if (best and second) else None

    return {
        "model_id": best[1].model_id if best else None,
        "best_key": best[0] if best else None,
        "best_loss": best[1].loss if best else None,
        "second_best_key": second[0] if second else None,
        "second_best_loss": second[1].loss if second else None,
        "margin": float(margin) if margin is not None else None,
        "scores": {k: {"loss": s.loss, "model_id": s.model_id} for k, s in scored},
        "errors": errors,
    }


**Evaluating warning handling**

I'm considering the use of forced_align, which is deprecated and may produce a warning. I'm not sure if we should suppress warnings, especially in the pipeline, since they might print each time at runtime and become noisy. Capturing warnings could be an option, but since this is a pipeline, repeated warnings seem like they could clutter the output. I'll need to find the best way to manage this without causing confusion.


**Implementing UI changes**

I need to ensure that our user interface is stable, so I’m focusing on making the schema reliable. The next step is to implement the necessary changes in ctc_fit, which should help with overall functionality. Alright, let’s get started on this patch. It feels like there’s a clear path ahead, and I’m eager to get everything running smoothly for the user!


Input:
*** Begin Patch
*** Update File: maya_transcribe/ctc_fit.py
@@
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any

 from maya_transcribe.audio import decode_pcm16
 from maya_transcribe.textnorm import normalize_text

 @dataclass(frozen=True)
 class CTCScore:
     model_id: str
     loss: float
     text: str
+    unk_ratio: float | None
+    target_len: int
+    token_score_mean: float | None
+    token_score_min: float | None
+    token_coverage: float | None
+    token_logp_threshold: float | None
+    span_count: int | None
+    span_mismatch: bool | None

 class _CTCModel:
@@
         if blank is None and getattr(self._processor, "tokenizer", None) is not None:
             blank = getattr(self._processor.tokenizer, "pad_token_id", None)
         self.blank_id = int(blank) if blank is not None else 0

-    def score(self, audio_path: Path, text: str, lang: str | None = None) -> float:
+    def score(
+        self,
+        audio_path: Path,
+        text: str,
+        lang: str | None = None,
+        *,
+        max_unk_ratio: float = 0.2,
+        token_logp_threshold: float = -3.0,
+    ) -> CTCScore:
         import torch.nn.functional as F
+        import warnings

         if not text:
             raise ValueError("empty_text")
         if "<unk>" in text or "<inaudible>" in text:
             raise ValueError("contains_unk")
@@
         # Tokenize targets.
         tokenizer = getattr(self._processor, "tokenizer", None)
         if tokenizer is None:
             raise ValueError("missing_tokenizer")
-        target_ids = tokenizer(text_norm).get("input_ids")
+        tokenized = tokenizer(text_norm, add_special_tokens=False)
+        target_ids = tokenized.get("input_ids")
         if not target_ids:
             raise ValueError("empty_targets")

+        unk_id = getattr(tokenizer, "unk_token_id", None)
+        unk_ratio: float | None = None
+        if unk_id is not None:
+            unk_count = sum(1 for tid in target_ids if int(tid) == int(unk_id))
+            unk_ratio = float(unk_count) / float(len(target_ids))
+            if unk_ratio > max_unk_ratio:
+                raise ValueError(f"too_many_unk_tokens:{unk_ratio:.3f}")
+
         # torch ctc_loss expects (T,N,C)
         log_probs_tnc = log_probs.transpose(0, 1)
         device = log_probs_tnc.device
         input_lengths = self._torch.tensor([log_probs_tnc.size(0)], dtype=self._torch.long, device=device)
         targets = self._torch.tensor(target_ids, dtype=self._torch.long, device=device)
         target_lengths = self._torch.tensor([targets.numel()], dtype=self._torch.long, device=device)
@@
             blank=self.blank_id,
             reduction="mean",
             zero_infinity=True,
         )
-        return float(loss.detach().cpu().item())
+        loss_value = float(loss.detach().cpu().item())
+
+        # Forced alignment token scores (coverage/fit). Best-effort: alignment may fail for short audio vs long text.
+        token_score_mean = None
+        token_score_min = None
+        token_coverage = None
+        span_count = None
+        span_mismatch = None
+
+        try:
+            import torchaudio
+
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore", message=".*deprecated.*forced_align.*", category=UserWarning)
+                aligned_tokens, aligned_scores = torchaudio.functional.forced_align(
+                    log_probs,
+                    targets.unsqueeze(0),
+                    input_lengths=input_lengths,
+                    target_lengths=target_lengths,
+                    blank=self.blank_id,
+                )
+
+            spans = torchaudio.functional.merge_tokens(
+                aligned_tokens[0].detach().cpu(), aligned_scores[0].detach().cpu(), blank=self.blank_id
+            )
+            span_scores = [float(s.score) for s in spans]
+            span_count = len(span_scores)
+            span_mismatch = (span_count != int(target_lengths.item()))
+            if span_scores:
+                token_score_mean = sum(span_scores) / float(len(span_scores))
+                token_score_min = min(span_scores)
+                token_coverage = sum(1 for s in span_scores if s >= token_logp_threshold) / float(len(span_scores))
+        except Exception:
+            pass
+
+        return CTCScore(
+            model_id=self.model_id,
+            loss=loss_value,
+            text=text,
+            unk_ratio=unk_ratio,
+            target_len=int(target_lengths.item()),
+            token_score_mean=token_score_mean,
+            token_score_min=token_score_min,
+            token_coverage=token_coverage,
+            token_logp_threshold=token_logp_threshold,
+            span_count=span_count,
+            span_mismatch=span_mismatch,
+        )
@@
 class CTCFitScorer:
@@
-    def __init__(self, model_map: dict[str, str] | None = None, default_model: str | None = None) -> None:
+    def __init__(
+        self,
+        model_map: dict[str, str] | None = None,
+        default_model: str | None = None,
+        *,
+        max_unk_ratio: float = 0.2,
+        token_logp_threshold: float = -3.0,
+    ) -> None:
         self.model_map = model_map or {}
         self.default_model = default_model
+        self.max_unk_ratio = float(max_unk_ratio)
+        self.token_logp_threshold = float(token_logp_threshold)
         self._cache: dict[str, _CTCModel] = {}
@@
     def score(self, audio_path: Path, text: str, lang: str | None) -> CTCScore | None:
         model_id = self._get_model_id(lang)
         if not model_id:
             return None
@@
-        loss = model.score(audio_path=audio_path, text=text, lang=lang)
-        return CTCScore(model_id=model_id, loss=loss, text=text)
+        return model.score(
+            audio_path=audio_path,
+            text=text,
+            lang=lang,
+            max_unk_ratio=self.max_unk_ratio,
+            token_logp_threshold=self.token_logp_threshold,
+        )
@@
     return {
         "model_id": best[1].model_id if best else None,
         "best_key": best[0] if best else None,
         "best_loss": best[1].loss if best else None,
         "second_best_key": second[0] if second else None,
         "second_best_loss": second[1].loss if second else None,
         "margin": float(margin) if margin is not None else None,
-        "scores": {k: {"loss": s.loss, "model_id": s.model_id} for k, s in scored},
+        "scores": {
+            k: {
+                "loss": s.loss,
+                "model_id": s.model_id,
+                "unk_ratio": s.unk_ratio,
+                "target_len": s.target_len,
+                "token_score_mean": s.token_score_mean,
+                "token_score_min": s.token_score_min,
+                "token_coverage": s.token_coverage,
+                "token_logp_threshold": s.token_logp_threshold,
+                "span_count": s.span_count,
+                "span_mismatch": s.span_mismatch,
+            }
+            for k, s in scored
+        },
         "errors": errors,
     }
*** End Patch


Output: {"output":"Success. Updated the following files:\nM maya_transcribe/ctc_fit.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Output:
Exit code: 0
Wall time: 0 seconds
Output:


**Updating verify.py**

I need to update verify.py to include new metrics and final_text. First, I'll take a look at the current state of verify.py so I can see what changes need to be made. By reviewing the code, I’ll get a clearer picture of where to incorporate the updates. This step will ensure I implement everything smoothly and maintain the intended functionality. Alright, let's get started on this!


sed -n '1,220p' maya_transcribe/verify.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

from dataclasses import dataclass
from typing import Any

from maya_transcribe.lid import LIDResult

@dataclass(frozen=True)
class Verification:
    decision: str  # "accept" | "human" | "reject"
    reasons: list[str]
    scores: dict[str, Any]

def lid_confidence(lid: LIDResult | None) -> dict[str, Any]:
    if not lid or not lid.predictions:
        return {"available": False, "top1": None, "top1_prob": None, "margin": None, "confident": False}
    top1 = lid.predictions[0]
    top2_prob = lid.predictions[1].prob if len(lid.predictions) > 1 else 0.0
    margin = float(top1.prob) - float(top2_prob)
    return {
        "available": True,
        "top1": top1.code,
        "top1_prob": float(top1.prob),
        "margin": margin,
        "confident": False,
    }

def verify_segment(
    *,
    qc: dict[str, Any],
    lid: LIDResult | None,
    final_text: str,
    rover: dict[str, Any],
    asr_summaries: dict[str, Any],
    primary_asr_key: str | None = None,
    ctc: dict[str, Any] | None = None,
    thresholds: dict[str, Any] | None = None,
) -> Verification:
    """
    Conservative verification gate (precision-first).
    """
    thresholds = thresholds or {}
    min_duration = float(thresholds.get("min_duration_sec", 0.8))
    max_clipping = float(thresholds.get("max_clipping_fraction", 0.01))
    max_disagreement = float(thresholds.get("max_rover_disagreement_rate", 0.03))
    lid_min_prob = float(thresholds.get("lid_min_prob", 0.90))
    lid_min_margin = float(thresholds.get("lid_min_margin", 0.20))
    require_lid = bool(thresholds.get("require_lid", True))

    # Whisper-style failure heuristics (see OpenAI whisper fallback logic).
    min_avg_logprob = float(thresholds.get("min_avg_logprob_mean", -1.0))
    max_compression_ratio = float(thresholds.get("max_compression_ratio_mean", 2.4))

    # CTC fit (audio-grounded) gating.
    require_ctc = bool(thresholds.get("require_ctc", False))
    min_ctc_margin = float(thresholds.get("min_ctc_margin", 0.0))

    reasons: list[str] = []
    decision = "accept"

    duration = float(qc.get("duration_sec") or 0.0)
    clipping = float(qc.get("clipping_fraction") or 0.0)
    if duration < min_duration:
        decision = "reject"
        reasons.append("too_short")
    if clipping > max_clipping:
        decision = "human" if decision != "reject" else decision
        reasons.append("clipping")

    lidc = lid_confidence(lid)
    if lidc["available"]:
        lidc["confident"] = bool(lidc["top1_prob"] >= lid_min_prob and lidc["margin"] >= lid_min_margin)
        if not lidc["confident"]:
            decision = "human" if decision != "reject" else decision
            reasons.append("lid_uncertain")
    else:
        reasons.append("lid_missing")
        if require_lid:
            decision = "human" if decision != "reject" else decision

    disagreement_rate = float(rover.get("disagreement_rate") or 0.0)
    if disagreement_rate > max_disagreement:
        decision = "human" if decision != "reject" else decision
        reasons.append("asr_disagreement")

    final_text = (final_text or "").strip()
    if "<unk>" in final_text or "<inaudible>" in final_text:
        decision = "human" if decision != "reject" else decision
        reasons.append("contains_unk")

    # Whisper-style safety: flag if any decoder claims "no speech" with high probability.
    no_speech_probs = []
    for summary in asr_summaries.values():
        p = summary.get("no_speech_prob_mean")
        if p is None:
            continue
        no_speech_probs.append(float(p))
    if no_speech_probs and max(no_speech_probs) > 0.8:
        decision = "reject"
        reasons.append("no_speech")

    # Whisper-style hallucination heuristics: apply to the primary ASR (or rover skeleton fallback).
    asr_key_for_heuristics = primary_asr_key or str(rover.get("skeleton_key") or "")
    heur = asr_summaries.get(asr_key_for_heuristics, {}) if asr_key_for_heuristics else {}
    avg_lp = heur.get("avg_logprob_mean")
    comp = heur.get("compression_ratio_mean")
    if avg_lp is not None and float(avg_lp) < min_avg_logprob:
        decision = "human" if decision != "reject" else decision
        reasons.append("low_logprob")
    if comp is not None and float(comp) > max_compression_ratio:
        decision = "human" if decision != "reject" else decision
        reasons.append("high_compression_ratio")

    # Audio-grounded fit check (when available).
    if require_ctc and (ctc is None or ctc.get("best_loss") is None):
        decision = "human" if decision != "reject" else decision
        reasons.append("ctc_missing")
    if ctc is not None:
        margin = ctc.get("margin")
        if margin is not None and float(margin) < min_ctc_margin:
            decision = "human" if decision != "reject" else decision
            reasons.append("ctc_ambiguous")

    scores = {
        "duration_sec": duration,
        "clipping_fraction": clipping,
        "rover_disagreement_rate": disagreement_rate,
        "lid": lidc,
        "no_speech_prob_max": max(no_speech_probs) if no_speech_probs else None,
        "heuristics_asr_key": asr_key_for_heuristics or None,
        "avg_logprob_mean": avg_lp,
        "compression_ratio_mean": comp,
        "ctc": ctc,
    }

    return Verification(decision=decision, reasons=reasons, scores=scores)


**Updating metrics and thresholds**

I want to include new metrics from the ctc, like token coverage, since verify.py only uses margin right now. I think we should update gating to factor in this coverage too, but I don’t want to make the default settings too strict—maybe we could set thresholds. I’ll suggest defaults like `min_ctc_token_coverage` at 0.85 and `min_ctc_token_score_mean` at -3.0. Also, should I set `fail_on_ctc_span_mismatch` to True and treat it as human? There’s also the “scores” dict to consider.


**Patching coverage thresholds**

I'm working on the patch. I'm setting up the thresholds for `min_ctc_token_coverage` at 0.85, `min_ctc_token_score_mean` at -3.0, and `fail_on_ctc_span_mismatch` as True by default. If the ctc is available, I’ll retrieve the best entry from the scores. For that entry, I need to check metrics like token coverage and mean score against the thresholds, appending reasons if they fall short. If `require_ctc` is True and metrics are missing, we should treat that as ambiguous and maybe introduce a threshold for `require_ctc_alignment_metrics`, defaulting to False.


**Patching unknown ratio handling**

Alright, I need to think about how to handle the `unk_ratio`. Even though we want to treat a high `unk_ratio` carefully, we’re already excluding candidates with high ratios. So, it seems I don't need to adjust that further. I’ll just make sure to patch verify.py to reflect this logic correctly. It’s important to keep the code clean and clear, ensuring it aligns with our intent! Let’s get this done!


Input:
*** Begin Patch
*** Update File: maya_transcribe/verify.py
@@
     # CTC fit (audio-grounded) gating.
     require_ctc = bool(thresholds.get("require_ctc", False))
     min_ctc_margin = float(thresholds.get("min_ctc_margin", 0.0))
+    min_ctc_token_coverage = float(thresholds.get("min_ctc_token_coverage", 0.85))
+    min_ctc_token_score_mean = float(thresholds.get("min_ctc_token_score_mean", -3.0))
+    fail_on_ctc_span_mismatch = bool(thresholds.get("fail_on_ctc_span_mismatch", True))

     reasons: list[str] = []
     decision = "accept"
@@
     if require_ctc and (ctc is None or ctc.get("best_loss") is None):
         decision = "human" if decision != "reject" else decision
         reasons.append("ctc_missing")
     if ctc is not None:
         margin = ctc.get("margin")
         if margin is not None and float(margin) < min_ctc_margin:
             decision = "human" if decision != "reject" else decision
             reasons.append("ctc_ambiguous")
+
+        best_key = ctc.get("best_key")
+        scores = ctc.get("scores")
+        if isinstance(best_key, str) and isinstance(scores, dict):
+            best = scores.get(best_key) or {}
+            if isinstance(best, dict):
+                cov = best.get("token_coverage")
+                if cov is not None and float(cov) < min_ctc_token_coverage:
+                    decision = "human" if decision != "reject" else decision
+                    reasons.append("ctc_low_coverage")
+                mean_tok = best.get("token_score_mean")
+                if mean_tok is not None and float(mean_tok) < min_ctc_token_score_mean:
+                    decision = "human" if decision != "reject" else decision
+                    reasons.append("ctc_low_token_score")
+                if fail_on_ctc_span_mismatch and bool(best.get("span_mismatch")):
+                    decision = "human" if decision != "reject" else decision
+                    reasons.append("ctc_span_mismatch")

     scores = {
         "duration_sec": duration,
         "clipping_fraction": clipping,
         "rover_disagreement_rate": disagreement_rate,
*** End Patch


Output: {"output":"Success. Updated the following files:\nM maya_transcribe/verify.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Output:
Exit code: 0
Wall time: 0 seconds
Output:


maya-transcribe run --segments-dir data/--0C5YVdu5s/--0C5YVdu5s/segments --out outputs/test_run_ctc2 --whisper-model tiny --limit 1 --ctc-model facebook/wav2vec2-base-960h


Output:
Exit code: 0
Wall time: 13.2 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-13 06:08:24.542501: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/wav2vec2-base-960h and are newly initialized: ['wav2vec2.masked_spec_embed']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Transcribing ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:03
Wrote: outputs/test_run_ctc2/results.jsonl
Wrote: outputs/test_run_ctc2/summary.json


python3 - <<'PY'
import json
from pathlib import Path
obj=json.loads(Path('outputs/test_run_ctc2/results.jsonl').read_text().splitlines()[0])
print('segment_id', obj['segment_id'])
print('decision', obj['verification']['decision'])
print('reasons', obj['verification']['reasons'])
ctc=obj.get('ctc')
print('ctc best', {k: ctc.get(k) for k in ['best_key','best_loss','margin']})
best_key=ctc.get('best_key')
print('ctc best metrics', ctc.get('scores',{}).get(best_key))
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
segment_id SPEAKER_00_0000_0.03-12.15-01b2a08391afc89b
decision human
reasons ['ctc_missing']
ctc best {'best_key': None, 'best_loss': None, 'margin': None}
ctc best metrics None


python3 - <<'PY'
import json
from pathlib import Path
obj=json.loads(Path('outputs/test_run_ctc2/results.jsonl').read_text().splitlines()[0])
print(obj.get('ctc'))
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
{'model_id': None, 'best_key': None, 'best_loss': None, 'second_best_key': None, 'second_best_loss': None, 'margin': None, 'scores': {}, 'errors': {'rover': 'ValueError:too_many_unk_tokens:0.875', 'faster-whisper:tiny|det': 'ValueError:too_many_unk_tokens:0.875', 'faster-whisper:tiny|samp_t0.2': 'ValueError:too_many_unk_tokens:0.875'}}


**Evaluating tokenizer behavior**

I’m contemplating how the tokenizer might treat spaces as unknown tokens because its vocabulary includes a ‘|’ character instead of a space. There’s a parameter `replace_word_delimiter_char=' '`, and I’m wondering if it’s configured to replace spaces with ‘|’. Even though the input text is already lowercase, the unknown ratio might be affected if the tokenizer expects uppercase.

I’m trying to sort out why it’s doing that, and if normalization to lowercase is playing a role here.


**Investigating tokenizer output**

I'm considering whether the `tokenizer(text_norm, add_special_tokens=False)` function returns token IDs for each character, along with including some special tokens. It seems that the `Wav2Vec2CTCTokenizer` uses ‘|’ as a word delimiter, which might impact how tokens are generated. I'll need to look into the vocabulary that's loaded from `vocab.json` to understand its structure better. This could clarify how the tokenizer operates and if it consistently handles special tokens alongside character tokens.


python3 - <<'PY'
from transformers import AutoProcessor
proc=AutoProcessor.from_pretrained('facebook/wav2vec2-base-960h')
tok=proc.tokenizer
print(type(tok))
print('unk_token', tok.unk_token, tok.unk_token_id)
print('pad_token', tok.pad_token, tok.pad_token_id)
print('vocab size', tok.vocab_size)
text='intricate web of'
enc=tok(text, add_special_tokens=False)
print('ids', enc['input_ids'][:30], 'len', len(enc['input_ids']))
unk_count=sum(1 for i in enc['input_ids'] if i==tok.unk_token_id)
print('unk_count', unk_count, 'ratio', unk_count/len(enc['input_ids']))
print('tokens', tok.convert_ids_to_tokens(enc['input_ids'][:30]))
PY


Output:
Exit code: 0
Wall time: 8.9 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-13 06:09:09.430681: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
<class 'transformers.models.wav2vec2.tokenization_wav2vec2.Wav2Vec2CTCTokenizer'>
unk_token <unk> 3
pad_token <pad> 0
vocab size 32
ids [3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3] len 16
unk_count 14 ratio 0.875
tokens ['<unk>', '<unk>', '<unk>', '<unk>', '<unk>', '<unk>', '<unk>', '<unk>', '<unk>', '|', '<unk>', '<unk>', '<unk>', '|', '<unk>', '<unk>']


Summary: auto


python3 - <<'PY'
from transformers import AutoProcessor
proc=AutoProcessor.from_pretrained('facebook/wav2vec2-base-960h')
tok=proc.tokenizer
vocab=tok.get_vocab()
# print sorted tokens
print('some vocab tokens:', sorted(vocab.keys())[:50])
print('contains a?', 'a' in vocab)
print('contains A?', 'A' in vocab)
print('contains i?', 'i' in vocab)
print('contains |?', '|' in vocab)
PY


Output:
Exit code: 0
Wall time: 9.7 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-13 06:09:25.318218: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
some vocab tokens: ["'", '</s>', '<pad>', '<s>', '<unk>', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '|']
contains a? False
contains A? True
contains i? False
contains |? True


**Adjusting Tokenization Normalization**

I've figured out that the vocabulary only contains uppercase letters, which sheds light on the unknown tokens problem. It's clear that I should uppercase English text before tokenization for this model. Lowercasing the text for CTC scoring seems detrimental to the tokenizer. I need to make adjustments, as my normalization process currently lowers the case for English. Perhaps we should modify the normalization for CTC models to keep everything uppercase. It’s typical to use uppercase in transcriptions for this tokenizer, after all.


**Adjusting Token Case Handling**

I can check if the tokenizer’s vocabulary includes 'A' but not 'a' to decide on uppercasing. It looks like using the `do_lower_case` attribute from the Wav2Vec2CTCTokenizer could help clarify this, as it’s probably set to false. Another method would be to adapt the case in `_CTCModel.score` after normalizing the text. Instead of calling `normalize_text` with a language parameter to lowercase, I could skip that to retain original case. If the vocabulary seems uppercase, I’ll transform the normalized text accordingly. Let's focus on determining if uppercasing is needed!


**Implementing Token Casing Logic**

For the Wav2Vec2CTCTokenizer, I can treat tokens as characters, special tokens, and the '|' symbol. I’ll create a method to apply casing based on the vocabulary: if 'a' is missing but 'A' is there, I’ll convert the text to uppercase. It seems wise to compute this only once in the `_CTCModel` initializer and store the result, instead of recalculating it each time. I’ll access the tokenizer from the processor during initialization to determine if uppercase adjustment is needed for scoring English text.


**Adjusting Uppercase Implementation**

It seems safe to convert English transcripts to uppercase, especially since some may include uppercase letters already. However, I need to be cautious with other languages, as uppercasing could lead to issues. Still, if the model’s vocabulary suggests that tokens are uppercase, then I can proceed with converting ASCII letters to uppercase while ensuring non-ASCII characters remain unchanged. I’ll update the `ctc_fit` to add an attribute `self._force_uppercase` based on the tokenizer's vocabulary, and then adjust `text_norm` accordingly if needed after normalization.


**Preserving Text Normalization**

I’m considering whether to call `normalize_text` with `lang=None` in order to preserve the case. However, since I plan to convert the text to uppercase anyway, I'm wondering if it’s necessary. It looks like calling with `lang=None` would avoid lowercasing the text. I’ll need to ensure that after applying `normalize_text`, the text can still be properly transformed to uppercase for scoring, so keeping that in mind will be important.


**Fixing scoring issues**

I’m noticing that for the pipeline route, I should convert the English text to lowercase first, and then to uppercase. It seems like this method will effectively patch the scoring for wav2vec2-base-960h. I think this approach will streamline the process, so I’m ready to implement this patch and see how it improves the scoring. Let’s go ahead and make that change!


Input:
*** Begin Patch
*** Update File: maya_transcribe/ctc_fit.py
@@
 class _CTCModel:
     def __init__(self, model_id: str, device: str | None = None) -> None:
@@
         self._processor = AutoProcessor.from_pretrained(model_id)
         self._model = AutoModelForCTC.from_pretrained(model_id).to(device)
         self._model.eval()

+        tokenizer = getattr(self._processor, "tokenizer", None)
+        self._force_uppercase = False
+        if tokenizer is not None and hasattr(tokenizer, "get_vocab"):
+            vocab = tokenizer.get_vocab()
+            # Common for wav2vec2 CTC: vocab contains A-Z but not a-z; transcripts should be uppercased.
+            self._force_uppercase = ("A" in vocab) and ("a" not in vocab)
+
         blank = getattr(self._model.config, "pad_token_id", None)
         if blank is None and getattr(self._processor, "tokenizer", None) is not None:
             blank = getattr(self._processor.tokenizer, "pad_token_id", None)
         self.blank_id = int(blank) if blank is not None else 0
@@
         # Conservative normalization helps tokenizer compatibility.
         text_norm = normalize_text(text, lang=lang)
         if not text_norm:
             raise ValueError("empty_text")
+        if self._force_uppercase:
+            text_norm = text_norm.upper()
*** End Patch


Output: {"output":"Success. Updated the following files:\nM maya_transcribe/ctc_fit.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Output:
Exit code: 0
Wall time: 0 seconds
Output:


maya-transcribe run --segments-dir data/--0C5YVdu5s/--0C5YVdu5s/segments --out outputs/test_run_ctc3 --whisper-model tiny --limit 1 --ctc-model facebook/wav2vec2-base-960h


Output:
Exit code: 0
Wall time: 13.1 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-13 06:10:17.170270: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/wav2vec2-base-960h and are newly initialized: ['wav2vec2.masked_spec_embed']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
/home/ubuntu/maya-transcribe/maya_transcribe/ctc_fit.py:149: UserWarning: torchaudio.functional._alignment.forced_align has been deprecated. This deprecation is part of a large refactoring effort to transition TorchAudio into a maintenance phase. Please see https://github.com/pytorch/audio/issues/3902 for more information. It will be removed from the 2.9 release.
  aligned_tokens, aligned_scores = torchaudio.functional.forced_align(
/home/ubuntu/maya-transcribe/maya_transcribe/ctc_fit.py:149: UserWarning: torchaudio.functional._alignment.forced_align has been deprecated. This deprecation is part of a large refactoring effort to transition TorchAudio into a maintenance phase. Please see https://github.com/pytorch/audio/issues/3902 for more information. It will be removed from the 2.9 release.
  aligned_tokens, aligned_scores = torchaudio.functional.forced_align(
/home/ubuntu/maya-transcribe/maya_transcribe/ctc_fit.py:149: UserWarning: torchaudio.functional._alignment.forced_align has been deprecated. This deprecation is part of a large refactoring effort to transition TorchAudio into a maintenance phase. Please see https://github.com/pytorch/audio/issues/3902 for more information. It will be removed from the 2.9 release.
  aligned_tokens, aligned_scores = torchaudio.functional.forced_align(
Transcribing ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:03
Wrote: outputs/test_run_ctc3/results.jsonl
Wrote: outputs/test_run_ctc3/summary.json


**Testing audio alignment**

I need to replicate a process in Python using `torchaudio.functional.forced_align`. I think I’ll call this function within `catch_warnings` to manage any potential warnings that might pop up. This should work well for what I'm trying to achieve. I want to keep the snippet clean and focused, ensuring it runs smoothly without any interruptions. Let’s pull it together and see how it performs!


python3 - <<'PY'
import warnings
import torch
import torchaudio

log_probs = torch.log_softmax(torch.randn(1, 50, 10), dim=-1)
targets = torch.randint(1, 9, (1, 10), dtype=torch.long)

with warnings.catch_warnings(record=True) as w:
    warnings.simplefilter('always')
    warnings.filterwarnings('ignore', message=r'.*forced_align.*deprecated.*', category=UserWarning)
    out = torchaudio.functional.forced_align(log_probs, targets, blank=0)
    print('captured', len(w))
    if w:
        print('first', type(w[0].message), str(w[0].message)[:120])
PY


Output:
Exit code: 0
Wall time: 1.7 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
captured 0


**Analyzing filter patterns**

The filter does work, but I’m wondering why there were warnings earlier. I think our filter pattern didn’t match the message pattern since it expected "deprecated" to appear before "forced_align". The warning message actually has "forced_align" before "deprecated", so it makes sense it didn’t match.

To fix this, we should update our pattern to either ".*forced_align.*deprecated.*" or just ".*deprecated.*". I’ll patch the code in `ctc_fit.py` to account for this. Looks like the warning is printed multiple times, possibly for each candidate scoring!


Input:
*** Begin Patch
*** Update File: maya_transcribe/ctc_fit.py
@@
             import torchaudio

             with warnings.catch_warnings():
-                warnings.filterwarnings("ignore", message=".*deprecated.*forced_align.*", category=UserWarning)
+                warnings.filterwarnings("ignore", message=".*forced_align.*deprecated.*", category=UserWarning)
                 aligned_tokens, aligned_scores = torchaudio.functional.forced_align(
                     log_probs,
                     targets.unsqueeze(0),
                     input_lengths=input_lengths,
                     target_lengths=target_lengths,
*** End Patch


Output: {"output":"Success. Updated the following files:\nM maya_transcribe/ctc_fit.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Output:
Exit code: 0
Wall time: 0 seconds
Output:


python3 - <<'PY'
import json
from pathlib import Path
obj=json.loads(Path('outputs/test_run_ctc3/results.jsonl').read_text().splitlines()[0])
print('decision', obj['verification']['decision'])
print('reasons', obj['verification']['reasons'])
ctc=obj['ctc']
print('ctc', {k: ctc.get(k) for k in ['best_key','best_loss','margin']})
best_key=ctc.get('best_key')
print('best metrics', ctc.get('scores',{}).get(best_key))
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
decision accept
reasons []
ctc {'best_key': 'rover', 'best_loss': 1.2419719696044922, 'margin': None}
best metrics {'loss': 1.2419719696044922, 'model_id': 'facebook/wav2vec2-base-960h', 'unk_ratio': 0.0, 'target_len': 16, 'token_score_mean': -0.4490797834514524, 'token_score_min': -6.753193378448486, 'token_coverage': 0.9375, 'token_logp_threshold': -3.0, 'span_count': 16, 'span_mismatch': False}


maya-transcribe gemini --audio outputs/test_run_ctc3/audio/SPEAKER_00_0000_0.03-12.15-01b2a08391afc89b.wav --language eng --model gemini-2.0-flash --max-output-tokens 64


Output:
Exit code: 0
Wall time: 3.2 seconds
Output:
full intricate web of


Input:
*** Begin Patch
*** Update File: maya_transcribe/pipeline.py
@@
                 for tf_backend in transformers_backends:
                     res = tf_backend.transcribe(wav_path, language=route_lang, num_beams=5, max_new_tokens=256)
                     asr_results[res.model] = res

                 # Gemini: optional extra hypothesis (prefer running only on "hard" segments).
@@

                 t_asr1 = time.perf_counter()

+                ensemble_models = sorted({v.model for v in asr_results.values()})
+                ensemble_backends = sorted({v.model.split(":", 1)[0] for v in asr_results.values()})
+                ensemble_warnings: list[str] = []
+                if len(ensemble_models) < 2:
+                    ensemble_warnings.append("low_model_diversity")
+
                 # 4) Normalize hypotheses
                 t_norm0 = time.perf_counter()
                 norm_hyps: dict[str, str] = {}
                 for k, res in asr_results.items():
                     lang_hint = res.language or route_lang
@@
                     "asr": {
                         k: {
                             "model": v.model,
                             "text": v.text,
                             "language": v.language,
                             "language_probability": v.language_probability,
                             "segments": [asdict(s) for s in v.segments],
                             "meta": v.meta,
                         }
                         for k, v in asr_results.items()
                     },
+                    "ensemble": {
+                        "unique_models": ensemble_models,
+                        "unique_backends": ensemble_backends,
+                        "warnings": ensemble_warnings,
+                    },
                     "normalized": norm_hyps,
                     "rover": {
                         "text": rover.text,
                         "skeleton_key": rover.skeleton_key,
                         "disagreement_rate": rover.disagreement_rate,
*** End Patch


Output: {"output":"Success. Updated the following files:\nM maya_transcribe/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Output:
Exit code: 0
Wall time: 0 seconds
Output:


maya-transcribe run --segments-dir data/--0C5YVdu5s/--0C5YVdu5s/segments --out outputs/test_run5 --whisper-model tiny --limit 1


Output:
Exit code: 0
Wall time: 11.1 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-13 06:12:00.498129: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
Transcribing ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:01
Wrote: outputs/test_run5/results.jsonl
Wrote: outputs/test_run5/summary.json


python3 - <<'PY'
import json
from pathlib import Path
obj=json.loads(Path('outputs/test_run5/results.jsonl').read_text().splitlines()[0])
print(obj['ensemble'])
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
{'unique_models': ['faster-whisper:tiny'], 'unique_backends': ['faster-whisper'], 'warnings': ['low_model_diversity']}


**Implementing changes in evaluation**

I think keeping the default parameter as `oracle: bool = True` is a good idea for evaluation. It might add some fields to the summary, which is essential since evaluation plays a role in tuning. But I'm aware that this might break earlier usage if someone was expecting different fields. Evaluating is important, and having references in the evaluation makes sense, so yeah, computing oracle is acceptable to support this. Let's move ahead!


Input:
*** Begin Patch
*** Update File: maya_transcribe/eval.py
@@
 @dataclass(frozen=True)
 class EvalRow:
     segment_id: str
     language: str | None
     decision: str | None
     duration_sec: float | None
     duration_bucket: str | None
     ref: str
     hyp: str
     wer: float
     cer: float
+    oracle_best_wer: float | None = None
+    oracle_best_cer: float | None = None
+    oracle_best_key: str | None = None
+    selection_gap_wer: float | None = None

 def load_refs(refs_path: Path) -> dict[str, dict[str, Any]]:
@@
 def evaluate(
     results_path: Path,
     refs_path: Path,
     out_path: Path | None = None,
     hypothesis: str = "final",
+    oracle: bool = True,
 ) -> dict[str, Any]:
@@
         ref_norm = normalize_text(ref_text, lang=lang)
         hyp_norm = normalize_text(hyp_text, lang=lang)
         w = float(wer(ref_norm, hyp_norm))
         c = float(cer(ref_norm.replace(" ", ""), hyp_norm.replace(" ", "")))

+        oracle_best_wer = None
+        oracle_best_cer = None
+        oracle_best_key = None
+        selection_gap_wer = None
+
+        if oracle:
+            cand_wer: dict[str, float] = {}
+            cand_cer: dict[str, float] = {}
+            for key, text in extract_all_candidates(result_row=r).items():
+                ref_n = ref_norm
+                hyp_n = normalize_text(text, lang=lang)
+                cand_wer[key] = float(wer(ref_n, hyp_n))
+                cand_cer[key] = float(cer(ref_n.replace(" ", ""), hyp_n.replace(" ", "")))
+
+            if cand_wer:
+                oracle_best_key, oracle_best_wer = min(cand_wer.items(), key=lambda kv: kv[1])
+                oracle_best_cer = cand_cer.get(oracle_best_key)
+                selection_gap_wer = float(w) - float(oracle_best_wer)
+
         rows.append(
             EvalRow(
                 segment_id=segment_id,
                 language=lang,
                 decision=str(decision) if decision is not None else None,
                 duration_sec=duration_sec,
                 duration_bucket=bucket,
                 ref=ref_text,
                 hyp=hyp_text,
                 wer=w,
                 cer=c,
+                oracle_best_wer=oracle_best_wer,
+                oracle_best_cer=oracle_best_cer,
+                oracle_best_key=oracle_best_key,
+                selection_gap_wer=selection_gap_wer,
             )
         )
@@
     by_lang: dict[str, list[float]] = {}
     by_dec: dict[str, list[float]] = {}
     by_bucket: dict[str, list[float]] = {}
     by_lang_cer: dict[str, list[float]] = {}
     by_dec_cer: dict[str, list[float]] = {}
     by_bucket_cer: dict[str, list[float]] = {}
+    selection_gaps: list[float] = []
+    oracle_best_wers: list[float] = []
     for row in rows:
         if row.language:
             by_lang.setdefault(row.language, []).append(row.wer)
             by_lang_cer.setdefault(row.language, []).append(row.cer)
         if row.decision:
             by_dec.setdefault(row.decision, []).append(row.wer)
             by_dec_cer.setdefault(row.decision, []).append(row.cer)
         if row.duration_bucket:
             by_bucket.setdefault(row.duration_bucket, []).append(row.wer)
             by_bucket_cer.setdefault(row.duration_bucket, []).append(row.cer)
+        if row.selection_gap_wer is not None:
+            selection_gaps.append(float(row.selection_gap_wer))
+        if row.oracle_best_wer is not None:
+            oracle_best_wers.append(float(row.oracle_best_wer))

     summary = {
         "hypothesis": hypothesis,
         "pairs_evaluated": len(rows),
         "wer_mean": (sum(r.wer for r in rows) / len(rows)) if rows else None,
         "cer_mean": (sum(r.cer for r in rows) / len(rows)) if rows else None,
@@
         "cer_by_language_mean": {k: sum(v) / len(v) for k, v in sorted(by_lang_cer.items())},
         "cer_by_decision_mean": {k: sum(v) / len(v) for k, v in sorted(by_dec_cer.items())},
         "cer_by_duration_bucket_mean": {k: sum(v) / len(v) for k, v in sorted(by_bucket_cer.items())},
+        "oracle": {
+            "enabled": bool(oracle),
+            "oracle_best_wer_mean": (sum(oracle_best_wers) / len(oracle_best_wers)) if oracle_best_wers else None,
+            "selection_gap_wer_mean": (sum(selection_gaps) / len(selection_gaps)) if selection_gaps else None,
+            "selection_gap_wer_p90": _percentile(selection_gaps, 0.90),
+        },
     }
@@
 def extract_hypothesis_text(result_row: dict[str, Any], hypothesis: str) -> str | None:
@@
     return None
+
+
+def extract_all_candidates(result_row: dict[str, Any]) -> dict[str, str]:
+    """
+    Collect all text candidates available in a pipeline result row.
+    Keys are stable and namespaced (final/rover/asr:<key>).
+    """
+    candidates: dict[str, str] = {}
+
+    final = extract_hypothesis_text(result_row, "final")
+    if final is not None:
+        candidates["final"] = final
+
+    rover = extract_hypothesis_text(result_row, "rover")
+    if rover is not None:
+        candidates["rover"] = rover
+
+    asr = result_row.get("asr") or {}
+    if isinstance(asr, dict):
+        for key, entry in asr.items():
+            if not isinstance(key, str):
+                continue
+            if not isinstance(entry, dict):
+                continue
+            text = entry.get("text")
+            if isinstance(text, str) and text.strip():
+                candidates[f"asr:{key}"] = text
+
+    # Also include normalized hypotheses if present (useful for debugging).
+    normalized = result_row.get("normalized") or {}
+    if isinstance(normalized, dict):
+        for key, text in normalized.items():
+            if not isinstance(key, str) or not isinstance(text, str):
+                continue
+            if text.strip():
+                candidates[f"norm:{key}"] = text
+
+    return candidates
*** End Patch


Output: {"output":"Success. Updated the following files:\nM maya_transcribe/eval.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Summary: auto


Input:
*** Begin Patch
*** Update File: maya_transcribe/cli.py
@@
 def eval(
     results: Path = typer.Option(..., exists=True, dir_okay=False, help="Pipeline results JSONL (outputs/run/results.jsonl)."),
     refs: Path = typer.Option(..., exists=True, dir_okay=False, help="Reference transcripts JSONL."),
     out: Path | None = typer.Option(None, help="Write summary JSON to this path."),
     hypothesis: str = typer.Option('final', help='Hypothesis to evaluate: "final", "rover", or "asr:<key>".'),
+    oracle: bool = typer.Option(True, help="Compute oracle-best and selection-gap stats across all candidates."),
 ) -> None:
@@
-    summary = evaluate(results_path=results, refs_path=refs, out_path=out, hypothesis=hypothesis)
+    summary = evaluate(results_path=results, refs_path=refs, out_path=out, hypothesis=hypothesis, oracle=oracle)
     typer.echo(json.dumps(summary, ensure_ascii=False, indent=2))
*** End Patch


Output: {"output":"Success. Updated the following files:\nM maya_transcribe/cli.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Output:
Exit code: 0
Wall time: 0 seconds
Output:


python3 - <<'PY'
import json
from pathlib import Path
obj=json.loads(Path('outputs/test_run_ctc3/results.jsonl').read_text().splitlines()[0])
seg=obj['segment_id']
text=obj['final']['text']
Path('outputs/test_run_ctc3/refs.jsonl').write_text(json.dumps({'segment_id':seg,'text':text},ensure_ascii=False)+'\n',encoding='utf-8')
print('refs ok', seg)
PY
maya-transcribe eval --results outputs/test_run_ctc3/results.jsonl --refs outputs/test_run_ctc3/refs.jsonl


Output:
Exit code: 0
Wall time: 0.3 seconds
Output:
refs ok SPEAKER_00_0000_0.03-12.15-01b2a08391afc89b
{
  "hypothesis": "final",
  "pairs_evaluated": 1,
  "wer_mean": 0.0,
  "cer_mean": 0.0,
  "wer_p50": 0.0,
  "wer_p90": 0.0,
  "wer_p95": 0.0,
  "cer_p50": 0.0,
  "cer_p90": 0.0,
  "cer_p95": 0.0,
  "segment_error_rate": 0.0,
  "pct_wer_gt_5pct": 0.0,
  "pct_wer_gt_10pct": 0.0,
  "wer_by_language_mean": {
    "eng": 0.0
  },
  "wer_by_decision_mean": {
    "accept": 0.0
  },
  "wer_by_duration_bucket_mean": {
    "0-3s": 0.0
  },
  "cer_by_language_mean": {
    "eng": 0.0
  },
  "cer_by_decision_mean": {
    "accept": 0.0
  },
  "cer_by_duration_bucket_mean": {
    "0-3s": 0.0
  },
  "oracle": {
    "enabled": true,
    "oracle_best_wer_mean": 0.0,
    "selection_gap_wer_mean": 0.0,
    "selection_gap_wer_p90": 0.0
  }
}


**Updating run plan statuses**

I should probably update the run plan statuses now. We previously used the update_plan tool, but since we've completed the plan earlier, there's a new one for CTC that we've already updated. In this second review, we can mention any changes even if we didn’t maintain the plan. The user didn’t specifically ask for a plan, but I'm confident we covered several important aspects, like stable segment IDs and other key metrics.


Input:
*** Begin Patch
*** Update File: Prompt.md
@@
 Code-switching:
 - Use the native script for the primary language.
+- If the language hint is unknown, detect the primary spoken language and use its native script.
 - If English words are spoken, keep them in Latin script as spoken (do not transliterate).
*** End Patch


Output: {"output":"Success. Updated the following files:\nM Prompt.md\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Ensuring command accuracy**

I've extracted the template, which is a good start. They mentioned a "Try It commands mismatch," so I need to ensure that our README or final response uses the correct CLI flags. We've updated the README with optional extras, but the command for the ctc-model is critical. It's also essential to address that the stable segment IDs have changed, meaning the existing refs.jsonl might not match. Lastly, there's a question about scores or metrics before creating the UI—considering a schema version in each record could help maintain stability in the output schema.


**Implementing schema versioning**

I'm wondering if it would be beneficial to implement `record["schema_version"]=1` and update the summary accordingly. Including configuration might also help streamline the process. I’ll move quickly here! In `pipeline.py`, I can add `schema_version: 1` at the top of the record. It may be worth doing something similar in the CLI preprocess as well. So, I'll work on patching that up to ensure everything is consistent. This should set a stable foundation.


Input:
*** Begin Patch
*** Update File: maya_transcribe/pipeline.py
@@
                 record = {
+                    "schema_version": 1,
                     "segment_id": segment_id,
                     "base_id": ident.base_id,
                     "relpath": ident.relpath,
                     "uid": ident.uid,
                     "src_path": str(src),
*** End Patch


Output: {"output":"Success. Updated the following files:\nM maya_transcribe/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Considering updates to the pipeline**

I'm thinking about whether I should update the preprocess rows for the UI. It seems like preprocessing could be optional, but I should definitely keep the pipeline results as the main focus. I want to make sure everything compiles well and that I’m capturing the important parts. Let's see how it all fits together and make sure I'm not missing anything crucial!


Output:
Exit code: 0
Wall time: 0 seconds
Output:


**Updating storage process**

I'm thinking it would be helpful to store which key is used for heuristics. I'll patch the record to include `asr_summary` as `asr_summaries`. In the pipeline.py, we compute `asr_summaries` before the verification step, so I'll store it there. It seems like this might enlarge the output a bit, but not by much. I believe this should work out fine!


Input:
*** Begin Patch
*** Update File: maya_transcribe/pipeline.py
@@
                 record = {
                     "schema_version": 1,
                     "segment_id": segment_id,
                     "base_id": ident.base_id,
                     "relpath": ident.relpath,
                     "uid": ident.uid,
                     "src_path": str(src),
                     "wav_path": str(wav_path),
                     "src_ffprobe": asdict(src_info),
                     "ffprobe": asdict(info),
                     "qc": asdict(qc),
                     "lid": (
                         None
                         if lid is None
                         else {
                             "model": lid.model,
                             "predictions": [{"code": p.code, "prob": p.prob} for p in lid.predictions],
                         }
                     ),
                     "route": {"language_hint": route_lang},
                     "asr": {
                         k: {
                             "model": v.model,
                             "text": v.text,
                             "language": v.language,
                             "language_probability": v.language_probability,
                             "segments": [asdict(s) for s in v.segments],
                             "meta": v.meta,
                         }
                         for k, v in asr_results.items()
                     },
+                    "asr_summary": asr_summaries,
                     "ensemble": {
                         "unique_models": ensemble_models,
                         "unique_backends": ensemble_backends,
                         "warnings": ensemble_warnings,
                     },
                     "normalized": norm_hyps,
                     "rover": {
                         "text": rover.text,
                         "skeleton_key": rover.skeleton_key,
                         "disagreement_rate": rover.disagreement_rate,
                         "disagreements": [
                             {"kind": d.kind, "index": d.index, "chosen": d.chosen, "options": d.options}
                             for d in rover.disagreements
                         ],
                     },
                     "ctc": ctc_info,
                     "final": {"text": final_text, "source": final_source},
                     "verification": {
                         "decision": verification.decision,
                         "reasons": verification.reasons,
                         "scores": verification.scores,
                     },
                     "config": {
                         "trim_silence": trim_silence,
                         "silence_threshold_db": silence_threshold_db,
                         "min_silence_sec": min_silence_sec,
                         "gemini_model": gemini_model,
                         "gemini_hard_only": gemini_hard_only,
                         "ctc_model": ctc_model,
                     },
                     "timing": {
                         "preprocess_sec": t_pre1 - t_pre0,
                         "lid_sec": t_lid1 - t_lid0,
                         "asr_sec": t_asr1 - t_asr0,
                         "normalize_sec": t_norm1 - t_norm0,
                         "rover_sec": t_rover1 - t_rover0,
                         "ctc_sec": t_ctc1 - t_ctc0,
                         "verify_sec": t_ver1 - t_ver0,
                         "total_sec": time.perf_counter() - t0,
                     },
                 }
*** End Patch


Output: {"output":"Success. Updated the following files:\nM maya_transcribe/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Output:
Exit code: 0
Wall time: 0 seconds
Output:


maya-transcribe run --segments-dir data/--0C5YVdu5s/--0C5YVdu5s/segments --out outputs/test_run6 --whisper-model tiny --limit 1


Output:
Exit code: 0
Wall time: 12.5 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-13 06:15:13.726985: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
Transcribing ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:01
Wrote: outputs/test_run6/results.jsonl
Wrote: outputs/test_run6/summary.json


python3 - <<'PY'
import json
from pathlib import Path
obj=json.loads(Path('outputs/test_run6/results.jsonl').read_text().splitlines()[0])
print('segment_id', obj['segment_id'])
print('schema_version', obj['schema_version'])
print('has asr_summary', 'asr_summary' in obj)
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
segment_id SPEAKER_00_0000_0.03-12.15-01b2a08391afc89b
schema_version 1
has asr_summary True


**Considering code adjustments**

I think we should adjust the CLI preprocess since it now consistently uses `segment_identity` as a hash. However, the function signature still includes an unused `seen` parameter, which may be confusing. We kept the parameter but didn’t use it; it might cause a linter flag, but it's not necessary to clean it up. Good news is we also updated `Prompt.md` with a new line! Now, I've got to ensure we address the environment variable mismatch for the user’s questions about scores or metrics before starting UI work.


**Proposing UI fields**

I'm thinking we should propose that the UI displays several key fields. First, audio info should include durations for both source and trimmed audio to show effects. Next, we could add quality control metrics like `rms_dbfs`, `peak_dbfs`, and `clipping_fraction`. For language identification, there should be top predictions with `lid.predictions` and verification scores. Ensemble warnings and all candidate transcripts are also useful. Lastly, displaying ROVER output, CTC scoring metrics, the final transcript, verification decision with reasons, and timing metrics could enhance user insight!


**Evaluating UI readiness**

I’m thinking if we’re ready for the UI, and I feel like we are! The pipeline output is machine-readable with stable segment IDs and includes the metrics needed for display and filtering. However, to improve WER quality, we still need to include Indic-specific models and select the right CTC models for each language. The UI is important for validation and calibration, and it could assist with manual corrections and reviews. I suggest creating a small web UI for features like audio playback and transcript comparisons.


**Checking for log security and updates**

I need to make sure we haven’t accidentally exposed any secrets in logs since we store raw output from Gemini, but it shouldn’t include keys, which is fine. Now, let’s look for any other corrections. The `ids.py` now uses sha1 prefix 16, which is good. We might also want to update the README to include info about segment IDs, changes, and ensure our pipeline run has `summary.json` with average latencies. I'll mention that caution is needed regarding the "ctc-model facebook/wav2vec2-base-960h" since it’s for English; users need to provide the correct CTC model map for other languages to avoid issues.


**Providing suggestions for improvements**

I need to suggest some improvements. First, we should mention that using MMS-LID returns ISO3 codes, which we then map to names. We changed `parse_strict_output` to allow extra outputs, but we still track if extra outputs are present without treating them as errors. We need to ensure that we pass the `language` parameter to `gemini_backend.transcribe` from the pipeline. The CLI command handles ISO3 or ISO2 codes and can return an auto-detect message if necessary. We've also fixed issues with segment IDs and added measures for CTC fit scoring to ensure proper tokenization. Additionally, verification metrics were enhanced with token coverage and mean score metrics.


**Addressing forced alignment concerns**

I'm realizing that `forced_align` is deprecated, but we've suppressed warnings for now. It might get removed in the future, so we could implement our own alignment algorithm or use alternatives like `torchaudio.functional._forced_align`. For robustness, it might be better to avoid relying on deprecated functions. We could calculate coverage using greedy decoded CER instead, which doesn't need forced alignment. If it fails, we can compute those metrics as a fallback. However, since `forced_align` might be removed, we should catch exceptions and skip metrics if necessary, allowing the pipeline to proceed without coverage.


**Planning for UI development**

I’m considering whether to treat missing metrics as optional but the review suggests that the coverage metric is important. To make the code robust, we could implement greedy CER as a fallback measure. In `_CTCModel.score`, instead of leaving metrics as None, we could calculate CER between greedy and target. I also need to address the review comment about alignment coverage. It seems we’re ready to start building the UI, having addressed stable segment IDs, language hints, and salvage parsing. We should clarify that real WER metrics aren't available yet due to the lack of a human gold set.


nl -ba maya_transcribe/ids.py | sed -n '1,120p'


rg -n "class ParsedGeminiOutput|def parse_strict_output" -n maya_transcribe/gemini_asr.py


rg -n "def to_prompt_language_hint" maya_transcribe/language.py


rg -n "max_unk_ratio|token_coverage|forced_align" maya_transcribe/ctc_fit.py


rg -n "ctc_low_coverage|min_ctc_token_coverage" maya_transcribe/verify.py


rg -n "oracle_best_wer_mean|extract_all_candidates" maya_transcribe/eval.py


rg -n "schema_version|asr_summary|ensemble" maya_transcribe/pipeline.py | head


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	This is a prompt for any **audio-native** model (Gemini, Voxtral, etc.) used as an extra ASR hypothesis.
     2	It is written to minimize hallucinations and keep outputs easy to validate.
     3
     4	`prompt`
     5
     6	`Instruction: You are a strict, verbatim transcription engine.
     7	Input: Audio file attached
     8	Language hint (primary): <language>
     9
    10	Task:
    11	- Transcribe only what is clearly spoken in the audio.
    12	- Do NOT guess or “fill in” missing words. If a word or phrase is unclear, output <unk> for that span.
    13	- Preserve repetitions, filler words, stutters, and false starts exactly as spoken.
    14	- Do NOT correct grammar, spelling, or pronunciation.
    15	- Do NOT translate or summarize.
    16	- Do NOT add punctuation.
    17
    18	Code-switching:
    19	- Use the native script for the primary language.
    20	- If the language hint is unknown, detect the primary spoken language and use its native script.
    21	- If English words are spoken, keep them in Latin script as spoken (do not transliterate).
    22
    23	OUTPUT FORMAT (exact):
    24	Output 1: Transcript
    25	<single transcript line>
    26
    27	Do not output anything else (no extra headings, notes, or additional outputs).`


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	import hashlib
     4	from dataclasses import dataclass
     5	from pathlib import Path
     6
     7
     8	@dataclass(frozen=True)
     9	class SegmentIdentity:
    10	    segment_id: str
    11	    base_id: str
    12	    relpath: str
    13	    uid: str
    14
    15
    16	def segment_identity(src: Path, root: Path, seen: dict[str, int] | None = None) -> SegmentIdentity:
    17	    """
    18	    Produce a stable segment identity and prevent collisions when stems repeat.
    19	    - `base_id`: `src.stem`
    20	    - `relpath`: path relative to `root` (or absolute as fallback)
    21	    - `uid`: sha1(relpath) prefix
    22	    - `segment_id`: always includes uid for stability across runs
    23	    """
    24	    base_id = src.stem
    25	    try:
    26	        relpath = src.relative_to(root).as_posix()
    27	    except Exception:
    28	        relpath = str(src)
    29
    30	    # Use 64 bits of hash to make collisions negligibly unlikely at large scale.
    31	    uid = hashlib.sha1(relpath.encode("utf-8")).hexdigest()[:16]
    32	    segment_id = f"{base_id}-{uid}"
    33
    34	    return SegmentIdentity(segment_id=segment_id, base_id=base_id, relpath=relpath, uid=uid)


Output:
Exit code: 0
Wall time: 0 seconds
Output:
79:class ParsedGeminiOutput:
100:def parse_strict_output(text: str) -> ParsedGeminiOutput:


Output:
Exit code: 0
Wall time: 0 seconds
Output:
80:def to_prompt_language_hint(lang_code: str | None) -> str:


Output:
Exit code: 0
Wall time: 0 seconds
Output:
20:    token_coverage: float | None
64:        max_unk_ratio: float = 0.2,
115:            if unk_ratio > max_unk_ratio:
140:        token_coverage = None
148:                warnings.filterwarnings("ignore", message=".*forced_align.*deprecated.*", category=UserWarning)
149:                aligned_tokens, aligned_scores = torchaudio.functional.forced_align(
166:                token_coverage = sum(1 for s in span_scores if s >= token_logp_threshold) / float(len(span_scores))
178:            token_coverage=token_coverage,
195:        max_unk_ratio: float = 0.2,
200:        self.max_unk_ratio = float(max_unk_ratio)
223:            max_unk_ratio=self.max_unk_ratio,
275:                "token_coverage": s.token_coverage,


Output:
Exit code: 0
Wall time: 0 seconds
Output:
60:    min_ctc_token_coverage = float(thresholds.get("min_ctc_token_coverage", 0.85))
136:                if cov is not None and float(cov) < min_ctc_token_coverage:
138:                    reasons.append("ctc_low_coverage")


Output:
Exit code: 0
Wall time: 0 seconds
Output:
111:            for key, text in extract_all_candidates(result_row=r).items():
185:            "oracle_best_wer_mean": (sum(oracle_best_wers) / len(oracle_best_wers)) if oracle_best_wers else None,
229:def extract_all_candidates(result_row: dict[str, Any]) -> dict[str, str]: