Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	from dataclasses import dataclass
     4	from pathlib import Path
     5	from typing import Any
     6
     7	from maya_transcribe.language import to_whisper_lang
     8
     9
    10	@dataclass(frozen=True)
    11	class ASRSegment:
    12	    start: float
    13	    end: float
    14	    text: str
    15	    avg_logprob: float | None = None
    16	    no_speech_prob: float | None = None
    17	    compression_ratio: float | None = None
    18
    19
    20	@dataclass(frozen=True)
    21	class ASRResult:
    22	    model: str
    23	    text: str
    24	    language: str | None
    25	    language_probability: float | None
    26	    segments: list[ASRSegment]
    27	    meta: dict[str, Any]
    28
    29
    30	class FasterWhisperASR:
    31	    def __init__(
    32	        self,
    33	        model_size_or_path: str = "large-v3",
    34	        device: str | None = None,
    35	        compute_type: str | None = None,
    36	    ) -> None:
    37	        try:
    38	            from faster_whisper import WhisperModel
    39	        except Exception as e:  # pragma: no cover
    40	            raise RuntimeError(
    41	                "faster-whisper is not installed. Install with: pip install -e '.[asr]'"
    42	            ) from e
    43
    44	        if device is None:
    45	            try:
    46	                import torch
    47
    48	                device = "cuda" if torch.cuda.is_available() else "cpu"
    49	            except Exception:
    50	                device = "cpu"
    51
    52	        if compute_type is None:
    53	            compute_type = "float16" if device == "cuda" else "float32"
    54
    55	        self.model_size_or_path = model_size_or_path
    56	        self.device = device
    57	        self.compute_type = compute_type
    58	        self._model = WhisperModel(model_size_or_path, device=device, compute_type=compute_type)
    59
    60	    @property
    61	    def name(self) -> str:
    62	        return f"faster-whisper:{self.model_size_or_path}"
    63
    64	    def transcribe(
    65	        self,
    66	        audio_path: Path,
    67	        language: str | None = None,
    68	        beam_size: int = 5,
    69	        best_of: int = 5,
    70	        temperature: float = 0.0,
    71	        vad_filter: bool = False,
    72	    ) -> ASRResult:
    73	        lang = to_whisper_lang(language)
    74
    75	        segments_iter, info = self._model.transcribe(
    76	            str(audio_path),
    77	            language=lang,
    78	            beam_size=beam_size,
    79	            best_of=best_of,
    80	            temperature=temperature,
    81	            vad_filter=vad_filter,
    82	        )
    83
    84	        segments: list[ASRSegment] = []
    85	        texts: list[str] = []
    86	        for s in segments_iter:
    87	            texts.append(s.text)
    88	            segments.append(
    89	                ASRSegment(
    90	                    start=float(s.start),
    91	                    end=float(s.end),
    92	                    text=str(s.text),
    93	                    avg_logprob=getattr(s, "avg_logprob", None),
    94	                    no_speech_prob=getattr(s, "no_speech_prob", None),
    95	                    compression_ratio=getattr(s, "compression_ratio", None),
    96	                )
    97	            )
    98
    99	        return ASRResult(
   100	            model=self.name,
   101	            text="".join(texts).strip(),
   102	            language=getattr(info, "language", None),
   103	            language_probability=getattr(info, "language_probability", None),
   104	            segments=segments,
   105	            meta={
   106	                "device": self.device,
   107	                "compute_type": self.compute_type,
   108	                "beam_size": beam_size,
   109	                "best_of": best_of,
   110	                "temperature": temperature,
   111	                "vad_filter": vad_filter,
   112	            },
   113	        )
   114
   115
   116	class TransformersWhisperASR:
   117	    """
   118	    Whisper-family ASR using HuggingFace Transformers (useful for fine-tuned checkpoints like IndicWhisper).
   119	    """
   120
   121	    def __init__(
   122	        self,
   123	        model_id: str,
   124	        device: str | None = None,
   125	        dtype: str | None = None,
   126	    ) -> None:
   127	        try:
   128	            import torch
   129	            from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
   130	        except Exception as e:  # pragma: no cover
   131	            raise RuntimeError(
   132	                "TransformersWhisperASR requires the optional dependencies: torch + transformers. "
   133	                "Install with: pip install -e '.[torch]'"
   134	            ) from e
   135
   136	        if device is None:
   137	            device = "cuda" if torch.cuda.is_available() else "cpu"
   138	        if dtype is None:
   139	            dtype = "float16" if (device == "cuda") else "float32"
   140
   141	        torch_dtype = torch.float16 if dtype == "float16" else torch.float32
   142
   143	        self.model_id = model_id
   144	        self.device = device
   145	        self.dtype = dtype
   146	        self._torch = torch
   147	        self._processor = AutoProcessor.from_pretrained(model_id)
   148	        self._model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id, torch_dtype=torch_dtype).to(device)
   149	        self._model.eval()
   150
   151	    @property
   152	    def name(self) -> str:
   153	        return f"transformers:{self.model_id}"
   154
   155	    def transcribe(
   156	        self,
   157	        audio_path: Path,
   158	        language: str | None = None,
   159	        max_new_tokens: int = 256,
   160	        num_beams: int = 5,
   161	    ) -> ASRResult:
   162	        from maya_transcribe.audio import decode_pcm16
   163
   164	        wav, sr = decode_pcm16(audio_path, sample_rate=16000, mono=True)
   165	        if wav.size == 0:
   166	            return ASRResult(
   167	                model=self.name,
   168	                text="",
   169	                language=language,
   170	                language_probability=None,
   171	                segments=[],
   172	                meta={"max_new_tokens": max_new_tokens, "num_beams": num_beams},
   173	            )
   174
   175	        inputs = self._processor(wav, sampling_rate=sr, return_tensors="pt")
   176	        input_features = inputs.get("input_features")
   177	        if input_features is None:
   178	            raise RuntimeError("Processor did not return input_features")
   179	        input_features = input_features.to(self.device)
   180
   181	        forced_decoder_ids = None
   182	        whisper_lang = to_whisper_lang(language)
   183	        if whisper_lang:
   184	            get_ids = getattr(self._processor, "get_decoder_prompt_ids", None)
   185	            if callable(get_ids):
   186	                forced_decoder_ids = get_ids(language=whisper_lang, task="transcribe")
   187
   188	        with self._torch.no_grad():
   189	            generated_ids = self._model.generate(
   190	                input_features,
   191	                forced_decoder_ids=forced_decoder_ids,
   192	                max_new_tokens=max_new_tokens,
   193	                num_beams=num_beams,
   194	            )
   195
   196	        text = self._processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
   197	        return ASRResult(
   198	            model=self.name,
   199	            text=text.strip(),
   200	            language=whisper_lang or language,
   201	            language_probability=None,
   202	            segments=[],
   203	            meta={
   204	                "device": self.device,
   205	                "dtype": self.dtype,
   206	                "max_new_tokens": max_new_tokens,
   207	                "num_beams": num_beams,
   208	            },
   209	        )


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Analyzing whitespace handling**

I'm wondering if our `normalize_text` actually removes spaces. It seems like it collapses whitespace but retains spaces. The pipeline constructs text by joining segments without spaces. In the `FasterWhisperASR.transcribe`, I see that it appends `s.text` to a list and then joins them together. If each segment's text starts with a leading space, joining them this way might still keep some spaces. However, if there's no leading space in `faster-whisper`, that could lead to every segment being crunched together, affecting the word error rate.


**Exploring WER causes**

I'm considering that the high word error rate (WER) might be linked to how we join segments. Using `''.join` could help, especially since both `faster-whisper` and OpenAI whisper have segments with leading spaces. I noted an example where a segment text starts with a space. I'll check this by transcribing short English audio if possible, to confirm. However, from the docs, it seems fine. The WER is computed on normalized text, which removes punctuation and collapses whitespace without adding new spaces. So, it shouldn't be an issue with segment joins affecting WER.


**Inspecting benchmark data**

I think it's a good idea to examine the benchmark corpora and evaluation results. They seem to have constructed the necessary data in the `data/bench` folder. I can check the `manifest.json` and `refs.jsonl` files to get insights into the languages and transcripts used. I'll also look in the `outputs/bench` directory for any evaluation results, particularly the `eval.json` file, which should contain WER metrics. It’ll help clarify things further!


set -euo pipefail
find data/bench -maxdepth 3 -type f -name manifest.json -o -name eval.json -o -name refs.jsonl | sed 's#^#- #'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
- data/bench/maya_hf_v1/refs.jsonl
- data/bench/maya_hf_v1/manifest.json
- data/bench/maya_hf_v2/refs.jsonl
- data/bench/maya_hf_v2/manifest.json
- data/bench/maya_hf_v0/refs.jsonl
- data/bench/maya_hf_v0/manifest.json


Summary: auto


**Inspecting corpora versions**

I’ve noted that there are three corpora: v0, v1, and v2. It seems that v0 is the initial version and v2 is deduped. V1 might be an intermediate version. I think I should take a closer look at the manifest for both v0 and v2 to better understand their differences and how they were structured. This will help clarify how they each play their role in the overall dataset.


set -euo pipefail
for d in data/bench/maya_hf_v0 data/bench/maya_hf_v2; do
  echo "--- $d/manifest.json"
  python3 - <<PY
import json
from pathlib import Path
p=Path('$d')/'manifest.json'
j=json.loads(p.read_text(encoding='utf-8'))
print('total_rows', j.get('total_rows'))
print('counts', j.get('counts'))
print('skipped', j.get('skipped'))
print('langs', j.get('langs')[:5], '...')
PY
done


Output:
Exit code: 0
Wall time: 0.1 seconds
Output:
--- data/bench/maya_hf_v0/manifest.json
total_rows 2400
counts {'asm': 200, 'ben': 200, 'eng': 200, 'guj': 200, 'hin': 200, 'kan': 200, 'mal': 200, 'mar': 200, 'ory': 200, 'pan': 200, 'tam': 200, 'tel': 200}
skipped {'asm': 1, 'ben': 0, 'eng': 0, 'guj': 0, 'hin': 0, 'kan': 0, 'mal': 0, 'mar': 0, 'ory': 0, 'pan': 0, 'tam': 0, 'tel': 0}
langs ['asm', 'ben', 'eng', 'guj', 'hin'] ...
--- data/bench/maya_hf_v2/manifest.json
total_rows 2400
counts {'asm': 200, 'ben': 200, 'eng': 200, 'guj': 200, 'hin': 200, 'kan': 200, 'mal': 200, 'mar': 200, 'ory': 200, 'pan': 200, 'tam': 200, 'tel': 200}
skipped {'asm': 1, 'ben': 0, 'eng': 0, 'guj': 0, 'hin': 0, 'kan': 0, 'mal': 0, 'mar': 0, 'ory': 0, 'pan': 0, 'tam': 0, 'tel': 0}
langs ['asm', 'ben', 'eng', 'guj', 'hin'] ...


**Verifying benchmarks**

I just checked that both v0 and v2 datasets have 2400 rows, which is 200 per language. That means the benchmark build succeeded! Now, they reported that the baseline run on v0 resulted in a WER mean of about 0.958. That's a specific value I need to consider. I should look for the outputs in the bench directory to dig deeper into this and see what further information or insights I can find!


set -euo pipefail
find outputs -maxdepth 4 -type f -name eval.json -o -name results.jsonl -o -name summary.json | sed 's#^#- #' | head -n 50


Output:
Exit code: 0
Wall time: 0 seconds
Output:
- outputs/bench/maya_hf_v0/odia_only_largev3/summary.json
- outputs/bench/maya_hf_v0/odia_only_largev3/results.jsonl
- outputs/bench/maya_hf_v0/odia_only_largev3/eval.json
- outputs/bench/maya_hf_v0/baseline/summary.json
- outputs/bench/maya_hf_v0/baseline/results.jsonl
- outputs/bench/maya_hf_v0/baseline/eval.json
- outputs/test_run6/summary.json
- outputs/test_run6/results.jsonl
- outputs/test_run3/summary.json
- outputs/test_run3/results.jsonl
- outputs/test_run_ctc3/summary.json
- outputs/test_run_ctc3/results.jsonl
- outputs/test_run_ctc2/summary.json
- outputs/test_run_ctc2/results.jsonl
- outputs/test_run4/summary.json
- outputs/test_run4/results.jsonl
- outputs/test_run2/summary.json
- outputs/test_run2/results.jsonl
- outputs/test_run_ctc/summary.json
- outputs/test_run_ctc/results.jsonl
- outputs/test_run/summary.json
- outputs/test_run/results.jsonl
- outputs/test_run5/summary.json
- outputs/test_run5/results.jsonl


set -euo pipefail
python3 - <<'PY'
import json
from pathlib import Path
p=Path('outputs/bench/maya_hf_v0/baseline/eval.json')
j=json.loads(p.read_text(encoding='utf-8'))
print(json.dumps({k:j.get(k) for k in ['pairs_evaluated','wer_mean','cer_mean','wer_p50','wer_p90','wer_p95','oracle']}, indent=2))
print('wer_by_language_mean (top):')
by=j.get('wer_by_language_mean') or {}
for k in sorted(by)[:20]:
    print(k, by[k])
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
{
  "pairs_evaluated": 2188,
  "wer_mean": 0.958137176179309,
  "cer_mean": 0.6892070914668146,
  "wer_p50": 1.0,
  "wer_p90": 1.2,
  "wer_p95": 1.6666666666666667,
  "oracle": {
    "enabled": true,
    "oracle_best_wer_mean": 0.8459395614677641,
    "selection_gap_wer_mean": 0.11219761471154517,
    "selection_gap_wer_p90": 0.11111111111111116
  }
}
wer_by_language_mean (top):
asm 1.2855066632628924
bak 1.0
ben 0.8297152641597085
ell 2.3333333333333335
eng 0.5861004342584378
guj 0.5656685455465943
hin 0.48697925451172186
hne 0.15384615384615385
kan 0.850996614108502
mal 1.3949920384779537
mar 1.0165202659994323
npi 1.3510926573426572
pan 0.836297166666294
pus 1.6709090909090907
rwr 0.4444444444444444
san 4.814814814814815
sin 0.8611111111111112
snd 0.75
som 1.0
tam 0.5683152987674263


**Analyzing language evaluation**

I'm looking at the `wer_by_language_mean`, and it includes more languages than our standard 12, which is intriguing. It seems `language_source` might actually refer to a route, not references. I notice some languages predicted by LID aren't from the actual list. It looks like there's a chance the baseline evaluation was done with different options than expected. The WER values vary significantly, suggesting potential issues with transcripts or mismatched references. So, it might affect the normalization process.


**Evaluating language inconsistencies**

I expected to see only 12 languages, but there are extra codes being used for grouping and normalization, which doesn't seem to significantly affect the WER. However, it's worth noting that WER can exceed 1.0, indicating more insertions than reference words. The overall WER mean is 0.958, but some languages fall short, while others, like mal, show very high values. Additionally, I found that 212 pairs are missing from evaluation, possibly due to references or transcripts not mapping correctly. The `san` language showing a WER of 4.81 is quite alarming and suggests LID might be mispredicting.


**Exploring language detection concerns**

I'm analyzing the baseline run on the bench corpus, where LID should be able to identify languages. However, audio segments are short, which might impact LID's accuracy. The route_lang setting is strict, requiring high confidence, leading to many instances where it’s set to None. Even when LID predicts a top language, there could still be errors. I realize that no language hint could lead to whisper outputs being incorrect, potentially raising the WER. It makes me wonder why LID remains uncertain so frequently.


**Analyzing LID output data**

I think route_lang should appear in the LID outputs, so it makes sense to check the baseline run results. I want to see how many segments had route_lang set, along with the distribution of the route languages and any relevant decisions. I’ll parse the `results.jsonl` file to gather this info.

I need to quickly run a Python script to count the segments. I’ll also look at the distribution of language hints, LID top codes, transcripts’ lengths, and any other details that catch my eye.


set -euo pipefail
python3 - <<'PY'
import json
from collections import Counter
from pathlib import Path

path = Path('outputs/bench/maya_hf_v0/baseline/results.jsonl')

counts_total=0
counts_route=Counter()
counts_lid=Counter()
counts_dec=Counter()
counts_reason=Counter()
empty_final=0
lang_by_ref=Counter()

# load refs to know true language
def load_refs():
    refs={}
    for line in Path('data/bench/maya_hf_v0/refs.jsonl').read_text(encoding='utf-8').splitlines():
        r=json.loads(line)
        refs[r['segment_id']]=r
    return refs
refs=load_refs()

for line in path.read_text(encoding='utf-8').splitlines():
    r=json.loads(line)
    counts_total += 1
    seg=r.get('segment_id')
    ref=refs.get(seg)
    if ref:
        lang_by_ref[ref.get('lang')] += 1
    route=(r.get('route') or {})
    rh=route.get('language_hint')
    if rh:
        counts_route[rh] += 1
    lid=(r.get('lid') or {})
    preds = lid.get('predictions') or []
    if preds:
        counts_lid[preds[0].get('code')] += 1
    ver=(r.get('verification') or {})
    dec=ver.get('decision')
    if dec:
        counts_dec[dec]+=1
    for reason in ver.get('reasons') or []:
        counts_reason[reason]+=1
    final=(r.get('final') or {})
    txt=(final.get('text') or '').strip()
    if not txt:
        empty_final += 1

print('total rows', counts_total)
print('true langs', lang_by_ref)
print('route_lang set', sum(counts_route.values()), '({:.1%})'.format(sum(counts_route.values())/counts_total))
print('top route langs', counts_route.most_common(10))
print('top lid top1 langs', counts_lid.most_common(10))
print('decision counts', counts_dec)
print('top reasons', counts_reason.most_common(15))
print('empty final', empty_final)
PY


Output:
Exit code: 0
Wall time: 0.2 seconds
Output:
total rows 2335
true langs Counter({'ben': 200, 'guj': 200, 'hin': 200, 'kan': 200, 'mal': 200, 'mar': 200, 'ory': 200, 'pan': 200, 'tam': 200, 'tel': 200, 'eng': 171, 'asm': 164})
route_lang set 1508 (64.6%)
top route langs [('tam', 188), ('guj', 164), ('asm', 151), ('tel', 150), ('pan', 149), ('mar', 144), ('kan', 143), ('mal', 142), ('ben', 135), ('hin', 77)]
top lid top1 langs [('guj', 206), ('tam', 200), ('tel', 182), ('pan', 172), ('mar', 171), ('asm', 166), ('kan', 165), ('ben', 157), ('mal', 156), ('hin', 135)]
decision counts Counter({'human': 1103, 'accept': 697, 'reject': 388})
top reasons [('asr_disagreement', 920), ('lid_uncertain', 540), ('too_short', 322), ('no_speech', 145), ('lid_missing', 140), ('high_compression_ratio', 118), ('clipping', 14), ('low_logprob', 2)]
empty final 303


set -euo pipefail
nl -ba maya_transcribe/pipeline.py | sed -n '280,460p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   280	                    ),
   281	                    "route": {"language_hint": route_lang},
   282	                    "asr": {
   283	                        k: {
   284	                            "model": v.model,
   285	                            "text": v.text,
   286	                            "language": v.language,
   287	                            "language_probability": v.language_probability,
   288	                            "segments": [asdict(s) for s in v.segments],
   289	                            "meta": v.meta,
   290	                        }
   291	                        for k, v in asr_results.items()
   292	                    },
   293	                    "asr_summary": asr_summaries,
   294	                    "ensemble": {
   295	                        "unique_models": ensemble_models,
   296	                        "unique_backends": ensemble_backends,
   297	                        "warnings": ensemble_warnings,
   298	                    },
   299	                    "normalized": norm_hyps,
   300	                    "rover": {
   301	                        "text": rover.text,
   302	                        "skeleton_key": rover.skeleton_key,
   303	                        "disagreement_rate": rover.disagreement_rate,
   304	                        "disagreements": [
   305	                            {"kind": d.kind, "index": d.index, "chosen": d.chosen, "options": d.options}
   306	                            for d in rover.disagreements
   307	                        ],
   308	                    },
   309	                    "ctc": ctc_info,
   310	                    "final": {"text": final_text, "source": final_source},
   311	                    "verification": {
   312	                        "decision": verification.decision,
   313	                        "reasons": verification.reasons,
   314	                        "scores": verification.scores,
   315	                    },
   316	                    "config": {
   317	                        "trim_silence": trim_silence,
   318	                        "silence_threshold_db": silence_threshold_db,
   319	                        "min_silence_sec": min_silence_sec,
   320	                        "gemini_model": gemini_model,
   321	                        "gemini_hard_only": gemini_hard_only,
   322	                        "ctc_model": ctc_model,
   323	                    },
   324	                    "timing": {
   325	                        "preprocess_sec": t_pre1 - t_pre0,
   326	                        "lid_sec": t_lid1 - t_lid0,
   327	                        "asr_sec": t_asr1 - t_asr0,
   328	                        "normalize_sec": t_norm1 - t_norm0,
   329	                        "rover_sec": t_rover1 - t_rover0,
   330	                        "ctc_sec": t_ctc1 - t_ctc0,
   331	                        "verify_sec": t_ver1 - t_ver0,
   332	                        "total_sec": time.perf_counter() - t0,
   333	                    },
   334	                }
   335
   336	                f.write(json.dumps(record, ensure_ascii=False) + "\n")
   337	                decision = verification.decision
   338	                counts[decision] = counts.get(decision, 0) + 1
   339	                timings_total.append(record["timing"]["total_sec"])
   340
   341	            except Exception as e:  # pragma: no cover
   342	                counts["error"] += 1
   343	                f.write(
   344	                    json.dumps(
   345	                        {"segment_id": segment_id, "src_path": str(src), "error": str(e)}, ensure_ascii=False
   346	                    )
   347	                    + "\n"
   348	                )
   349
   350	    summary = {
   351	        "segments_total": len(audio_files),
   352	        "counts": counts,
   353	        "avg_total_sec": (sum(timings_total) / len(timings_total)) if timings_total else None,
   354	        "p50_total_sec": _percentile(timings_total, 0.50),
   355	        "p90_total_sec": _percentile(timings_total, 0.90),
   356	    }
   357	    summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
   358	    return results_path
   359
   360
   361	def _percentile(xs: list[float], q: float) -> float | None:
   362	    if not xs:
   363	        return None
   364	    xs_sorted = sorted(xs)
   365	    idx = int(round((len(xs_sorted) - 1) * q))
   366	    idx = max(0, min(len(xs_sorted) - 1, idx))
   367	    return float(xs_sorted[idx])


**Analyzing evaluation errors**

I'm evaluating some errors in my process. The records include segment IDs, but it seems evaluation may have skipped some rows without these IDs. The decisions add up to 2188, matching the pairs evaluated, but I noticed some segments are missing. The expected total is 2400, yet the results show only 2335 rows. I'm wondering if the pipeline was interrupted or if there was a limit causing it to stop early. It's definitely puzzling!


**Investigating incomplete processing**

I’m thinking that the baseline run likely got interrupted, processing only 2335 segments, which aligns with 2188 evaluation pairs when excluding errors. When analyzing language counts, I noticed 'eng' processed 171 and 'asm' only 164, while others had 200 each. It seems the pipeline sorts audio files alphabetically. So it completed all segments for 'ben' but stopped early with 'eng' and 'asm'. This makes me wonder if it halted right near the beginning of 'eng.'


**Clarifying inconsistencies in processing**

I’m noticing some inconsistencies here. If the sorting properly puts the 'asm' folder first and the run stopped early, then it shouldn't have processed later languages like 'tel.' However, it did manage to process 200 for 'tel' and others. It seems some segments in 'asm' and 'eng' might have had errors and were skipped, yet still logged in error records. It suggests those segments might be missing from the audio directory. Although the manifest states there should be 200 files for each, I’m wondering if the build was incomplete.


**Analyzing segment counts**

I'm trying to understand why we only counted 164 'asm' and 171 'eng' segments. In my analysis, I loaded reference mappings and tallied the segments found in the results file, including errors. This indicates the pipeline only processed 164 'asm' and 171 'eng' segments, leaving 65 missing segments, which matches the difference between 2400 and 2335. It seems the pipeline might have encountered an exception, but it should have logged an error record for any issues. This means the results file should align with the number of audio files processed.


**Investigating segment processing issues**

I'm wondering about the segment_id variable defined inside the try block. However, segment_id is set before that, so even if an exception occurs, it should still write an error record. Each audio source should have a line unless the program crashed or was interrupted, leading to an incomplete results file. It seems like the baseline run could have been aborted, leaving many segments processed. The WER mean of 0.958 indicates poor performance overall, although Hindi's WER of 0.49 isn’t terrible. I should check the evaluation summary next to see if accepted segments have lower WER.


set -euo pipefail
python3 - <<'PY'
import json
from pathlib import Path
j=json.loads(Path('outputs/bench/maya_hf_v0/baseline/eval.json').read_text(encoding='utf-8'))
print('wer_by_decision_mean', j.get('wer_by_decision_mean'))
print('cer_by_decision_mean', j.get('cer_by_decision_mean'))
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
wer_by_decision_mean {'accept': 0.6600290144894885, 'human': 1.068130605528151, 'reject': 1.1809687125866046}
cer_by_decision_mean {'accept': 0.3739248809149772, 'human': 0.7492229748256926, 'reject': 1.0849652909765737}


Summary: auto


**Analyzing WER statistics**

The average Word Error Rate (WER) for accepted segments is 0.66, which is still high but an improvement. For human segments, the average is 1.07, and rejected segments show 1.18. It seems gating correlates with these errors but not effectively. I discovered that the default model in use is "large-v3" from CLI with faster-whisper, which should yield good transcripts. High WER could stem from long dataset transcripts or issues with punctuation, but the Hindi WER is surprisingly low at 0.49.


**Evaluating WER computation**

I’m starting to think that we might be evaluating the Word Error Rate (WER) incorrectly. It’s computed on normalized text, but there could be whitespace segmentation issues in different languages. Hindi has spaces, and a WER of 0.49 seems reasonable for moderate alignment, yet Gujarati is at 0.56, which isn’t great. This could stem from audio and text mismatches due to bugs in dataset construction. I should inspect specific segments and consider running `ffprobe` to check if the audio matches the transcripts. This could help clarify the situation!


**Investigating WER issues**

In the `bench.py` file for building the benchmark corpus, it looks like the text and audio are properly aligned since `ex = ds[i]` confirms that. So, why is the WER so high? It might be due to using the Whisper large-v3 model, but there’s a chance it’s not downloaded or that a smaller model is being used instead. The `whisper_model` parameter should bring in `large-v3` from the ctranslate2 repository. However, if they used a smaller model, that could explain the high WER. I should check the summary.json file to investigate further.


set -euo pipefail
cat outputs/bench/maya_hf_v0/baseline/summary.json


Output:
Exit code: 0
Wall time: 0 seconds
Output:
{
  "segments_total": 2335,
  "counts": {
    "accept": 697,
    "human": 1103,
    "reject": 388,
    "error": 147
  },
  "avg_total_sec": 2.8313835532605816,
  "p50_total_sec": 2.681691577658057,
  "p90_total_sec": 4.6828044638969
}


**Evaluating audio files count**

I'm noticing that the summary suggests there are 2335 segments processed instead of the expected 2400, which could mean some audio files are missing from the build. It seems the audio_files list initially had 2335 files, but the benchmark directory should include 2400. Perhaps duplicates or path collisions could be involved due to how file names were created. If file names overlapped, it would lead to a lower total than anticipated, and I suspect that might be the issue!


**Investigating audio file discrepancies**

I'm realizing the summary indicates there are 2335 segments, matching the number of audio files found during the bench run. If there were file name collisions, it would result in fewer files while still showing 2400 reference rows since those are added without checking for duplicates. This could mean many references point to overwritten or missing files. The 65 missing segments likely connect to these collisions. I wonder if they fixed this in the v2 version? I need to verify audio file counts in v0 and v2.


set -euo pipefail
for v in v0 v2; do
  echo "--- maya_hf_$v"
  echo -n "audio files: "
  find data/bench/maya_hf_${v}/audio -type f -name '*.wav' | wc -l
  echo -n "refs rows: "
  wc -l < data/bench/maya_hf_${v}/refs.jsonl
  echo -n "unique segment_ids in refs: "
  python3 - <<PY
import json
from pathlib import Path
ids=[]
for line in Path('data/bench/maya_hf_${v}/refs.jsonl').read_text(encoding='utf-8').splitlines():
    ids.append(json.loads(line)['segment_id'])
print(len(set(ids)))
PY
  echo

done


Output:
Exit code: 0
Wall time: 0.1 seconds
Output:
--- maya_hf_v0
audio files: 2335
refs rows: 2400
unique segment_ids in refs: 2335

--- maya_hf_v2
audio files: 2400
refs rows: 2400
unique segment_ids in refs: 2400


**Analyzing audio file duplicates**

I've discovered that in version 0, there are 2,335 audio files but 2,400 rows in references, which suggests 65 duplicates in segment IDs due to collisions or overwrites. Each segment ID matches a file name, so those duplicates mean I've lost 65 audio files. This seems crucial. They mentioned that a stable segment ID should always be based on the base ID and UID. Still, collisions might occur if based on safe name collisions. I'm considering checking ids.py for more clarity.


set -euo pipefail
nl -ba maya_transcribe/ids.py | sed -n '1,200p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
     1	from __future__ import annotations
     2
     3	import hashlib
     4	from dataclasses import dataclass
     5	from pathlib import Path
     6
     7
     8	@dataclass(frozen=True)
     9	class SegmentIdentity:
    10	    segment_id: str
    11	    base_id: str
    12	    relpath: str
    13	    uid: str
    14
    15
    16	def segment_identity(src: Path, root: Path) -> SegmentIdentity:
    17	    """
    18	    Produce a stable segment identity and prevent collisions when stems repeat.
    19	    - `base_id`: `src.stem`
    20	    - `relpath`: path relative to `root` (or absolute as fallback)
    21	    - `uid`: sha1(relpath) prefix
    22	    - `segment_id`: always includes uid for stability across runs
    23	    """
    24	    base_id = src.stem
    25	    try:
    26	        relpath = src.relative_to(root).as_posix()
    27	    except Exception:
    28	        relpath = str(src)
    29
    30	    # Use 64 bits of hash to make collisions negligibly unlikely at large scale.
    31	    uid = hashlib.sha1(relpath.encode("utf-8")).hexdigest()[:16]
    32	    segment_id = f"{base_id}-{uid}"
    33
    34	    return SegmentIdentity(segment_id=segment_id, base_id=base_id, relpath=relpath, uid=uid)


**Evaluating filename collisions**

I'm looking at filename collisions in the bench build. The safe_name function might create identical filenames for different examples because it incorporates src_key__i__src_id. But it looks like safe_name could be collapsing too aggressively, especially if src_id includes special characters or gets trimmed. The i index is supposed to be unique per dataset, but if they shuffle across sources, it could repeat. I wonder if safe_name is stripping too much, potentially leading to path length issues. Duplicates seem to have only appeared for asm and eng due to the shared FLEURS dataset.


**Analyzing potential collisions**

I’m considering the role of the index, i, within the dataset since it’s unique. Even if src_id collides, i should differentiate them. So, how can collisions still occur? Examining safe_name, it replaces certain characters and can ultimately map different strings to the same name. Collisions should only happen if the same i and src_key are used; however, there's no repetition in the selected example indices. Even across sources, the split names in src_key keep things unique, so collisions shouldn’t happen.


**Exploring duplicate issues**

I'm noticing that duplicates occurred, possibly because `wav_path` uses `lang_dir`, which is decided by the language code. Within a language, the unique fname should also be unique. However, there were duplicates with asm and eng. Safe_name doesn't truncate, but maybe filename length restrictions in the filesystem come into play. The OS may throw an error instead of truncating. Duplicates could suggest that files were overwritten rather than writing errors. I wonder if safe_name modified i to become the same string for different indices. But since i is an integer, it should yield different fnames.


**Understanding the duplicate issue**

I’m exploring why duplicates can’t happen due to differences in i. Unless i is very large and safe_name trims trailing underscores, but that doesn’t seem to be the case. The existence of segment_id 2335 vs. 2400 suggests that 65 duplicates were written to the same wav_path. This could occur if the fname produced the same output for different i values. However, since i remains as an integer and is used in the f-string correctly, it should be unique. There's a possibility of a bug, but it seems like the indices are handled as integers.


**Investigating duplicate causes**

It seems duplicates might not arise from fname collisions, but rather from the segment_identity function that generates segment_id based on wav_path relative to audio_root. If wav_path is unique, then segment_id should be too. However, that suggests file path collisions occurred, particularly in build_benchmark_corpus for v0. The v2 deduplication likely fixed these issues, as it created 2400 unique segment_ids. This means that the baseline evaluation for v0 might show inflated WER due to duplicate entries in the refs, which would misalign with audio files. After deduplication, WER in v2 could be lower. I should check the evaluation for the v2 run.


set -euo pipefail
find outputs/bench -maxdepth 3 -type f -name eval.json | sed 's#^#- #'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
- outputs/bench/maya_hf_v0/odia_only_largev3/eval.json
- outputs/bench/maya_hf_v0/baseline/eval.json


**Proposing ASR Checkpoints**

The user is asking for my input on which ASR checkpoints to use next, suggesting options for either single multilingual or per-language IDs. I need to keep my response concise but also provide enough guidance. I’m thinking it might be helpful to mention a robust evaluation methodology for the pipeline, including dataset selection like AI4Bharat Vistaar benchmarks versus HF corpora. Since the user previously asked for ideas, they might want to discuss model selection further. I'll check the bench harness for potential issues.


**Proposing Modifications for Robustness**

I’m considering some modifications to improve robustness. The current `bench_build` uses `load_dataset` in a non-streaming way, which can be heavy for large datasets. For the validation split, it may still download the whole dataset, so we might want to use streaming or specify splits instead. It loads audio arrays directly, which is manageable but still a bit weighty—especially for a large number of samples. The pipeline could benefit from resampling to 16k to save space, although it’s not necessary. Overall, we seem to be on the right track.


**Evaluating Potential Issues**

There’s a potential issue with the FLEURS dataset where the audio array might be in float32 format and the sampling rate is possibly 48,000. When writing to WAV at this rate, the pipeline will resample to 16k, which is fine. The segments directory uses `rglob` to find audio files without recursion, but it works well for per-language subdirectories. However, we need to evaluate LID gating closely, as there may be many segments marked as human due to ASR disagreements; discrepancies can lead to unnecessary human decisions even when transcripts are correct.


**Reassessing ASR Disagreement Evaluation**

It seems their evaluation of ASR disagreement might be too strict, showing a disagreement in 920 segments out of 2,188, which is significant. This might happen because they treat detection and sampling as separate, but they’re actually the same model. It might make more sense to compute disagreement only across distinct models, not multiple passes of the same one. Also, adjusting thresholds for ROVER might be better. The gating affects decisions about acceptance while the final transcript remains crucial. While aiming to minimize WER, I’d be cautious about how ROVER combines transcripts, as it could degrade quality.


**Evaluating ASR Options and Recommendations**

It looks like they’re currently using detection and sampling from the same model, which might not yield the best results. Instead, I suggest using just a deterministic model or incorporating different model sizes for more diversity. The sampling might just add noise and could degrade WER. They’re also using specific parameters for Whisper that could lead to random outputs, so I wonder if a temperature of 0 might be more effective.

For the next ASR models, especially for Indic languages, I recommend looking into AI4Bharat IndicWhisper, potentially with various sizes as options. I’ll search quickly on HF for model IDs to finalize my recommendation.


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=ai4bharat/indicwhisper&limit=50" | python3 - <<'PY'
import sys, json
j=json.load(sys.stdin)
for m in j:
    print(m.get('modelId'))
PY


Output:
Exit code: 1
Wall time: 0.2 seconds
Output:
Traceback (most recent call last):
  File "<stdin>", line 2, in <module>
  File "/usr/lib/python3.10/json/__init__.py", line 293, in load
    return loads(fp.read(),
  File "/usr/lib/python3.10/json/__init__.py", line 346, in loads
    return _default_decoder.decode(s)
  File "/usr/lib/python3.10/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/usr/lib/python3.10/json/decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)


Summary: auto


**Investigating API failure**

I'm looking into why the API failed. It might be that the response came back in HTML format because of rate limiting. I need to check the headers to understand more about what happened. Maybe there are some clues there that will help me figure out the problem. I'll proceed with printing the header to get that information, so I can figure out the next steps to resolve this issue.


set -euo pipefail
curl -s -D - "https://huggingface.co/api/models?search=ai4bharat/indicwhisper&limit=5" | head


Output:
Exit code: 23
Wall time: 0.2 seconds
Output:
HTTP/2 200
content-type: application/json; charset=utf-8
content-length: 1429
date: Tue, 13 Jan 2026 15:53:11 GMT
etag: W/"595-c63hpzJzGSlG5Jpj9aQZhkTo/LI"
x-powered-by: huggingface-moon
x-request-id: Root=1-69666a67-39ac5edc5d959e2824de3a2a
ratelimit: "api";r=498;t=85
ratelimit-policy: "fixed window";"api";q=500;w=300
cross-origin-opener-policy: same-origin


**Revising Python use**

I’ve realized we can’t use `python3` with heredoc and pipe at the same time because both are trying to read from stdin. In earlier commands, we used `python3 -c` for parsing, so I think it's best to stick with that. I’ll redo this to ensure it works correctly. It's sometimes tricky to manage input methods, but clarity helps in getting the right output!


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=ai4bharat/indicwhisper&limit=50" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'


Output:
Exit code: 0
Wall time: 0.3 seconds
Output:
ai4bharat/indicwav2vec_v1_gujarati
ai4bharat/indicwav2vec_v1_hindi
ai4bharat/indicwav2vec_v1_marathi
ai4bharat/indicwav2vec_v1_nepali
ai4bharat/indicwav2vec_v1_odia
ai4bharat/indicwav2vec_v1_sinhala
ai4bharat/indicwav2vec_v1_tamil
ai4bharat/indicwav2vec_v1_telugu
ai4bharat/indicwav2vec_v1_bengali
ai4bharat/indicwav2vec-hindi
ai4bharat/indicwav2vec-odia


**Searching for IndicWhisper models**

I noticed that my search for "IndicWhisper" returned results related to indicwav2vec, but I'm not entirely sure this is the right connection. I wonder if the AI4Bharat IndicWhisper models might be listed under different names, like "ai4bharat/indic-whisper" or "ai4bharat/whisper-large-v2". It might be a good idea to search for "IndicWhisper" in a more general way to clarify things. Let's see what comes up!


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=IndicWhisper&limit=50" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'


Output:
Exit code: 0
Wall time: 0.2 seconds
Output:
sawradip/indicwhisper-m-hi-float16-ct2
parthiv11/indic_whisper_hi_multi_gpu
parthiv11/indic_whisper_nodcil
Vignesh-M/Indic-whisper
HariprasathSB/indic-whisper-vulnerable
HariprasathSB/indic-whisper-vulnerable1
HariprasathSB/indic-whisper-vulnerable2
vrclc/indicwhisper-Mal
kavyamanohar/AI4B-Indicwhisper-ml
kavyamanohar/AI4B-Indicwhisper-or
kritianandan/indic-whisper-kn-reverse-mft-1-0-1-8
kritianandan/indic-whisper-kn-reverse-mft-1-0-1-9
kritianandan/indic-whisper-kn-reverse-mft-1-0-1-10
kritianandan/indic-whisper-reverse-ml-mft-1-0-1-11
kritianandan/indic-whisper-reverse-ml-mft-1-0-1-13
kritianandan/indic-whisper-reverse-ml-mft-1-0-1-14
Koosh0610/indic-whisper-reverse-ml-mft-1-0-1-15
adalat-ai/ct2-AI4B-Indicwhisper-ml


**Searching for AI4Bharat resources**

I see some search results indicating fine-tuned models, but I need to find out more about AI4Bharat's IndicWhisper. It might not be on Hugging Face or another organization. I wonder if AI4Bharat has a specific project for "indic-whisper," but it's not showing up in my results. So, I think my next step is to search specifically for "ai4bharat whisper" to gather more accurate information.


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=ai4bharat%2Fwhisper&limit=50" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'


Output:
Exit code: 0
Wall time: 0.2 seconds
Output:
soketlabs/whisper_indic_hi_ai4bharat


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=ct2-AI4B-Indicwhisper&limit=50" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'


Output:
Exit code: 0
Wall time: 0.2 seconds
Output:
adalat-ai/ct2-AI4B-Indicwhisper-ml


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=AI4B-Indicwhisper&limit=100" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'


Output:
Exit code: 0
Wall time: 0.2 seconds
Output:
kavyamanohar/AI4B-Indicwhisper-ml
kavyamanohar/AI4B-Indicwhisper-or
adalat-ai/ct2-AI4B-Indicwhisper-ml


**Exploring ASR Models**

I'm considering various approaches for using the best available ASR models per language. For instance, models like `google/whisper-large-v3` could work well. There are options like `Systran/faster-whisper-large-v3` and various `ai4bharat` models for languages such as Gujarati, Hindi, and Tamil, covering many nuances. Some languages might need special attention, and I wonder if outputs can be decoded to native scripts. Checking the gating status for these models could also provide valuable insights. Let’s see if we can access the models.


set -euo pipefail
for id in ai4bharat/indicwav2vec_v1_hindi ai4bharat/indicwav2vec_v1_telugu ai4bharat/indicwav2vec_v1_tamil ai4bharat/indicwav2vec_v1_bengali ai4bharat/indicwav2vec_v1_odia; do
  echo "--- $id"
  curl -s "https://huggingface.co/api/models/$id" | python3 -c "import sys,json; j=json.load(sys.stdin); print('private=',j.get('private'), 'gated=', j.get('gated'), 'tags_has_asr=', 'automatic-speech-recognition' in (j.get('tags') or []))"
done


Output:
Exit code: 0
Wall time: 0.8 seconds
Output:
--- ai4bharat/indicwav2vec_v1_hindi
private= False gated= auto tags_has_asr= False
--- ai4bharat/indicwav2vec_v1_telugu
private= False gated= auto tags_has_asr= False
--- ai4bharat/indicwav2vec_v1_tamil
private= False gated= auto tags_has_asr= False
--- ai4bharat/indicwav2vec_v1_bengali
private= False gated= auto tags_has_asr= True
--- ai4bharat/indicwav2vec_v1_odia
private= False gated= auto tags_has_asr= False


**Reviewing Whisper models**

I’m considering adding some Whisper-family models. There are community fine-tuned variants for specific Indic languages, like `vasista22/whisper-hindi-small`. However, the user asked about what Hugging Face repositories are needed to access these datasets, not models. They provided a summary for review, so I should give feedback on that. It might be a good idea to recommend using `openai/whisper-large-v3` along with at least one more model. For their requirements, perhaps a multilingual option like `distil-whisper/distil-large-v3` would suit them best!


**Exploring language models**

I’m thinking about selecting the best available models for various languages. For Hindi, there are several options, and for Tamil, we might do a similar approach. I can suggest using "whisper-large-v3" along with CTC model `facebook/mms-1b-all` for scoring and decoding transcripts as a fallback. They're already using CTC scoring, so maybe MMS could work as an ASR model. They also mentioned code modifications, but since the user didn't ask for changes, my suggestion for implementing a `CTCASR` backend might just be an option to propose!


**Addressing dataset access and reviews**

The user previously asked for dataset suggestions and a code review. Now they're looking for Hugging Face repositories to access these datasets before running benchmarking tests. It seems they've already created a bench harness and possibly want another review.

I need to comment on the design of the bench harness, highlighting potential issues like segment_id collisions and the importance of running on version 2. I’ll also suggest checking `bench_build` for gated datasets to confirm access, plus recommending an ablation matrix to evaluate performance under different configurations.


**Finding offline ASR models**

I’m exploring options to add offline ASR capabilities with fine-tuned Whisper models. I initially thought about trying `sawradip/indicwhisper-m-hi-float16-ct2`, but that’s for CTranslate2 and not suitable for Transformers. I should search for fine-tuned Whisper models on Hugging Face that can run offline, although they may require significant compute power.

I’m considering models like `openai/whisper-large-v3` and possibly `vasista22/whisper-hindi-large-v2` for Hindi. However, I also want to check if there's a multi-language model like "Vignesh-M/Indic-whisper" that could fit our needs better!


set -euo pipefail
for id in Vignesh-M/Indic-whisper parthiv11/indic_whisper_hi_multi_gpu sawradip/indicwhisper-m-hi-float16-ct2; do
  echo "--- $id"
  curl -s "https://huggingface.co/api/models/$id" | python3 -c "import sys,json; j=json.load(sys.stdin); print('gated=', j.get('gated'), 'private=', j.get('private')); print('pipeline_tag=', j.get('pipeline_tag')); print('tags=', (j.get('tags') or [])[:10])"
done


Output:
Exit code: 0
Wall time: 0.5 seconds
Output:
--- Vignesh-M/Indic-whisper
gated= False private= False
pipeline_tag= automatic-speech-recognition
tags= ['transformers', 'pytorch', 'whisper', 'automatic-speech-recognition', 'endpoints_compatible', 'region:us']
--- parthiv11/indic_whisper_hi_multi_gpu
gated= False private= False
pipeline_tag= automatic-speech-recognition
tags= ['transformers', 'pytorch', 'jax', 'whisper', 'automatic-speech-recognition', 'music', 'hi', 'license:mit', 'endpoints_compatible', 'deploy:azure']
--- sawradip/indicwhisper-m-hi-float16-ct2
gated= False private= False
pipeline_tag= None
tags= ['region:us']


Summary: auto


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=whisper%20telugu&pipeline_tag=automatic-speech-recognition&limit=20" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'


Output:
Exit code: 0
Wall time: 0.3 seconds
Output:
ARTPARK-IISc/whisper-medium-vaani-telugu
steja/whisper-small-telugu
steja/whisper-small-telugu-large-data
vasista22/whisper-telugu-tiny
vasista22/whisper-telugu-base
vasista22/whisper-telugu-small
vasista22/whisper-telugu-medium
vasista22/whisper-telugu-large-v2
Harsha9044/whisper-tiny-fluers_V1_telugu
eswardivi/whisper-tiny-fluers_V2_telugu_Augmentation_bf16
eswardivi/whisper-tiny-fluers_V2_telugu_Augmentation_full_datset_V2_e5
nurenora/whisper-finetune-telugu
AKAWIZ/whisper_telugu_medium
kowshik/whisper-telugu-large-v2
kowshik/whisper-telugu-medium
Mukund017/whisper-small-telugu
jayasuryajsk/whisper-large-v3-Telugu-Romanized
heisenberg3376/whisper-small-finetuned-telugu
Anujgr8/Whisper-Anuj-small-Telugu-final
kaarthu2003/whisper-large-telugu


**Evaluating ASR models**

I'm looking at various models for offline use, and for each language, the user wants minimal Word Error Rate (WER) possible. To achieve this, we can consider ensemble and verification strategies, but the base ASR quality is essential. I’m thinking we should select fine-tuned Whisper models from reputable sources like ARTPARK for multiple Indian languages, such as "whisper-medium-vaani-telugu" and "whisper-medium-vaani-hindi." Those seem to be high-quality options to consider.


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=ARTPARK-IISc/whisper-medium-vaani&limit=50" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'


Output:
Exit code: 0
Wall time: 0.3 seconds
Output:
ARTPARK-IISc/whisper-medium-vaani-telugu
ARTPARK-IISc/whisper-small-vaani-kannada
ARTPARK-IISc/whisper-small-vaani-tulu
ARTPARK-IISc/whisper-small-vaani-hindi
ARTPARK-IISc/whisper-medium-vaani-kannada
ARTPARK-IISc/whisper-medium-vaani-hindi
ARTPARK-IISc/whisper-medium-vaani-bengali
ARTPARK-IISc/whisper-tiny-vaani-hindi
ARTPARK-IISc/whisper-large-v3-vaani-hindi
ARTPARK-IISc/whisper-large-v3-vaani-telugu
ARTPARK-IISc/whisper-large-v3-vaani-odia


set -euo pipefail
ids=(
  'ARTPARK-IISc/whisper-large-v3-vaani-hindi'
  'ARTPARK-IISc/whisper-large-v3-vaani-telugu'
  'ARTPARK-IISc/whisper-large-v3-vaani-odia'
  'ARTPARK-IISc/whisper-medium-vaani-bengali'
  'ARTPARK-IISc/whisper-medium-vaani-kannada'
  'ARTPARK-IISc/whisper-medium-vaani-hindi'
  'Vignesh-M/Indic-whisper'
  'openai/whisper-large-v3'
)
for id in "${ids[@]}"; do
  echo "--- $id"
  curl -s "https://huggingface.co/api/models/$id" | python3 -c "import sys,json; j=json.load(sys.stdin); print('gated=',j.get('gated'),'private=',j.get('private'),'pipeline_tag=',j.get('pipeline_tag'))"
done


Output:
Exit code: 0
Wall time: 2.6 seconds
Output:
--- ARTPARK-IISc/whisper-large-v3-vaani-hindi
gated= False private= False pipeline_tag= automatic-speech-recognition
--- ARTPARK-IISc/whisper-large-v3-vaani-telugu
gated= False private= False pipeline_tag= None
--- ARTPARK-IISc/whisper-large-v3-vaani-odia
gated= False private= False pipeline_tag= None
--- ARTPARK-IISc/whisper-medium-vaani-bengali
gated= False private= False pipeline_tag= automatic-speech-recognition
--- ARTPARK-IISc/whisper-medium-vaani-kannada
gated= False private= False pipeline_tag= automatic-speech-recognition
--- ARTPARK-IISc/whisper-medium-vaani-hindi
gated= False private= False pipeline_tag= automatic-speech-recognition
--- Vignesh-M/Indic-whisper
gated= False private= False pipeline_tag= automatic-speech-recognition
--- openai/whisper-large-v3
gated= False private= False pipeline_tag= automatic-speech-recognition


**Choosing models for languages**

All the models are ungated, which is great. I’ve noticed that the pipeline tag for ARTPARK in Telugu and Odia is set to None; it might still use ASR, but that’s not confirmed. Now, I need to pick models for other languages like Tamil, Malayalam, Marathi, and so on. For instance, Whisper large-v3 seems good for English, and for Hindi, ARTPARK large-v3 vaaNi could be better. I'll keep exploring for Tamil models too.


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=vaani%20tamil&limit=20" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'


Output:
Exit code: 0
Wall time: 0.3 seconds
Output:


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=whisper%20tamil%20large-v2&pipeline_tag=automatic-speech-recognition&limit=20" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'


Output:
Exit code: 0
Wall time: 0.2 seconds
Output:
vasista22/whisper-tamil-large-v2
kowshik/whisper-tamil-large-v2
JairamKanna/whisper-large-v2-tamil


set -euo pipefail
for id in vasista22/whisper-tamil-large-v2 kowshik/whisper-tamil-large-v2; do
  echo "--- $id"
  curl -s "https://huggingface.co/api/models/$id" | python3 -c "import sys,json; j=json.load(sys.stdin); print('gated=',j.get('gated'),'private=',j.get('private'),'pipeline_tag=',j.get('pipeline_tag'))"
done


Output:
Exit code: 0
Wall time: 1.5 seconds
Output:
--- vasista22/whisper-tamil-large-v2
gated= False private= False pipeline_tag= automatic-speech-recognition
--- kowshik/whisper-tamil-large-v2
gated= False private= False pipeline_tag= automatic-speech-recognition


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=whisper%20malayalam%20large&pipeline_tag=automatic-speech-recognition&limit=20" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'


Output:
Exit code: 0
Wall time: 1.3 seconds
Output:
kurianbenoy/whisper_malayalam_largev2
DrishtiSharma/whisper-large-v2-malayalam
CXDuncan/whisper-large-v3-malayalam


set -euo pipefail
for id in kurianbenoy/whisper_malayalam_largev2 CXDuncan/whisper-large-v3-malayalam kavyamanohar/AI4B-Indicwhisper-ml; do
  echo "--- $id"
  curl -s "https://huggingface.co/api/models/$id" | python3 -c "import sys,json; j=json.load(sys.stdin); print('gated=',j.get('gated'),'private=',j.get('private'),'pipeline_tag=',j.get('pipeline_tag'))"
done


Output:
Exit code: 0
Wall time: 0.8 seconds
Output:
--- kurianbenoy/whisper_malayalam_largev2
gated= False private= False pipeline_tag= automatic-speech-recognition
--- CXDuncan/whisper-large-v3-malayalam
gated= False private= False pipeline_tag= automatic-speech-recognition
--- kavyamanohar/AI4B-Indicwhisper-ml
gated= False private= False pipeline_tag= None


set -euo pipefail
for lang in marathi gujarati punjabi assamese; do
  echo "=== $lang"
  curl -s "https://huggingface.co/api/models?search=whisper%20${lang}%20large&pipeline_tag=automatic-speech-recognition&limit=10" | python3 -c 'import sys,json; j=json.load(sys.stdin); print("\n".join(m.get("modelId","") for m in j))'
  echo
 done


Output:
Exit code: 0
Wall time: 0.9 seconds
Output:
=== marathi
DrishtiSharma/whisper-large-v2-marathi
durgesh10/whisper-large-marathi
durgesh10/whisper-large-v3-marathi
khushi1234455687/whisper-large-v3-turbo-Marathi-Version1
KhushiDS/whisper_large_v3_Marathi

=== gujarati
khushi1234455687/Whisper-large-v3-Gujarati
khushi1234455687/Whisper-large-v3-Gujarati-New

=== punjabi
DrishtiSharma/whisper-large-v2-punjabi-700-steps
DrishtiSharma/whisper-large-v2-punjabi
khushi1234455687/whisper-large-v3-Punjabi-Version1
khushi1234455687/Whisper-large-v3-Punjabi-Version1-New

=== assamese
DrishtiSharma/whisper-large-v2-assamese-700-steps
DrishtiSharma/whisper-large-v2-assamese-5k-steps
khushi1234455687/whisper-large-v3-Assamese-Version1
khushi1234455687/Whisper-large-v3-Assamese-Version1-New
khushi1234455687/whisper-large-v3-turbo-Assamese-Version1
hblogan/whisper-large-v3-turbo-assamese


set -euo pipefail
ids=(
  'durgesh10/whisper-large-v3-marathi'
  'DrishtiSharma/whisper-large-v2-punjabi'
  'DrishtiSharma/whisper-large-v2-assamese-5k-steps'
  'khushi1234455687/Whisper-large-v3-Gujarati'
)
for id in "${ids[@]}"; do
  echo "--- $id"
  curl -s "https://huggingface.co/api/models/$id" | python3 -c "import sys,json; j=json.load(sys.stdin); print('gated=',j.get('gated'),'private=',j.get('private'),'pipeline_tag=',j.get('pipeline_tag')); print('tags_has_whisper=', 'whisper' in (j.get('tags') or []))"
done


Output:
Exit code: 0
Wall time: 1.5 seconds
Output:
--- durgesh10/whisper-large-v3-marathi
gated= False private= False pipeline_tag= automatic-speech-recognition
tags_has_whisper= True
--- DrishtiSharma/whisper-large-v2-punjabi
gated= False private= False pipeline_tag= automatic-speech-recognition
tags_has_whisper= True
--- DrishtiSharma/whisper-large-v2-assamese-5k-steps
gated= False private= False pipeline_tag= automatic-speech-recognition
tags_has_whisper= True
--- khushi1234455687/Whisper-large-v3-Gujarati
gated= auto private= False pipeline_tag= automatic-speech-recognition
tags_has_whisper= False


**Searching for Gujarati models**

I'm thinking about the Gujarati model being gated and that the user might need access. It could be worthwhile to check out an ungated version instead. I wonder if there are other models from khushi that might also be gated, or if there are additional options I can find. I should do a more thorough search for ungated Gujarati models to help the user out! Let's see what I can discover.


set -euo pipefail
curl -s "https://huggingface.co/api/models?search=whisper%20gujarati&pipeline_tag=automatic-speech-recognition&limit=30" | python3 -c 'import sys,json; j=json.load(sys.stdin);
for m in j:
    print(m.get("modelId"), "gated="+str(m.get("gated")))'


Output:
Exit code: 0
Wall time: 0.2 seconds
Output:
vasista22/whisper-gujarati-base gated=None
vasista22/whisper-gujarati-small gated=None
vasista22/whisper-gujarati-medium gated=None
kowshik/whisper-gujarati-medium gated=None
DevOG/whisper-gujarati-medium gated=None
khushi1234455687/Whisper-large-v3-Gujarati gated=None
khushi1234455687/Whisper-large-v3-Gujarati-New gated=None
haideraqeeb/whisper-gujarati-finetuned gated=None
KhushiDS/whisper-lare-v3-gujarati gated=None


set -euo pipefail
for id in vasista22/whisper-gujarati-medium kowshik/whisper-gujarati-medium DevOG/whisper-gujarati-medium; do
  echo "--- $id"
  curl -s "https://huggingface.co/api/models/$id" | python3 -c "import sys,json; j=json.load(sys.stdin); print('gated=',j.get('gated'),'private=',j.get('private'),'pipeline_tag=',j.get('pipeline_tag'))"
done


Output:
Exit code: 0
Wall time: 0.6 seconds
Output:
--- vasista22/whisper-gujarati-medium
gated= False private= False pipeline_tag= automatic-speech-recognition
--- kowshik/whisper-gujarati-medium
gated= False private= False pipeline_tag= automatic-speech-recognition
--- DevOG/whisper-gujarati-medium
gated= False private= False pipeline_tag= automatic-speech-recognition


**Considering model selection**

If a model is fine-tuned for a specific language, it should ideally still support others. However, managing 12 transformer models for each segment isn't practical. Instead, per-language model selection could streamline this by running only the relevant fine-tuned model based on the reference language. Implementing code adjustments to allow mapping from language to a specific model ID would enhance efficiency. Alternatively, a single multilingual model like Vignesh-M/Indic-whisper could reduce compute needs. I can propose a two-phase approach to test this idea.


**Addressing model improvements**

If needed, I could add per-language models for languages with poor word error rates like Malayalam, Assamese, and Punjabi—this seems manageable. The user is asking about the required Hugging Face repositories for datasets, suggesting they're checking if the code is robust enough to move forward with benchmarking. They’re summarizing improvements and likely want my review to confirm if everything is correct and which next steps to take. I should provide comments on discovered issues, ensuring that evaluations are aligned with the reference language and update benchmarks accordingly.


**Validating code improvements**

They've already applied fixes in ctc_fit and language mapping, and added a benchmark harness. It looks like they've built corpora versions 0 and 2 and run baselines. I think they want us to validate the robustness of the code—like confirming if the ctc target language fix is accurate and whether the benchmark harness is safe. Examining bench.py, it seems the deduplication might not be evident. Although v2 has unique audio files, there’s a question about how that uniqueness was achieved in build_benchmark_corpus if it doesn't actually deduplicate.


**Clarifying dataset access**

I'm trying to figure out why v0 had collisions; it seems that the earlier version of bench.py didn't include `i` or src_key, only using src_id which likely caused those collisions. It sounds like they updated bench.py later, releasing v2 with a more robust design. Great! They've also added optional dependencies in pyproject and new CLI bench commands. With the user's request regarding dataset access, I need to clarify which Hugging Face repositories they need for those datasets and potentially for ASR checkpoints as well.


**Preparing ASR recommendations**

I notice a new message about key fixes, and while the user might not have specific questions, it’s a good idea to provide another review and suggest next ASR checkpoints. I’ll gather a list of recommended Hugging Face model repositories for them to try, along with an ablation plan, though I want to avoid using the plan tool unnecessarily since the user hasn't explicitly asked for it. I also need to confirm that the ctc_fit fix is both correct and robust, so I’ll quickly run `python -m compileall maya_transcribe` to check for syntax errors.


set -euo pipefail
python3 -m compileall -q maya_transcribe


Output:
Exit code: 0
Wall time: 0 seconds
Output:


Summary: auto


**Reviewing code robustness**

I’m starting to review the code for robustness, particularly concerning edge cases for benchmarking. The LID gating has strict thresholds and requires LID to be true, marking many segments as human, which is acceptable for production but might not be ideal for benchmarking ASR WER without gating. Thankfully, the addition of thresholds_json lets us adjust that. I need to ensure it affects the route_lang decision in the pipeline, and I’ll also verify the correct application of thresholds. Lastly, pipeline settings default to require_ctc being true, which is another potential issue to consider.


**Examining CTC requirements and CLI updates**

I see that the verification process requires CTC availability, but CTC can fail with some languages, potentially routing those to human review, which is fine. For benchmarking runs, CTC shouldn't be required since it's optional — they can set `require_ctc` to false in thresholds_json. However, there's a bit of confusion about the CLI commands; it mentions "bench run accept," but that command doesn't exist. They’ve added support for datasets and huggingface_hub, which is great, but we need to be cautious about dataset licensing when pushing. Additionally, the evaluation process can now group by reference language.


**Clarifying HF repos and dataset access**

I’m considering which HF repositories the user might need for datasets, and I already have a list of dataset repos like `ai4bharat/Kathbath`. The user might also be interested in checking access for models. I could anticipate a question about the private HF dataset `maya-research/maya-transcribe-bench-v2`, but they haven’t asked about it yet. Looking back at their message, they mentioned some key fixes related to benchmarking and specifics about ASR checkpoints for their next attempt. They seem to be aiming for a more detailed analysis with an ablation matrix to address WER improvements.


**Suggesting ASR checkpoints**

The user is asking which ASR checkpoints to try next, and they want to consult their other agent about this. We need to point out that the baseline evaluation on version 0 is invalid due to collisions, so version 2 should be used, and we should also compute WER by language and decision. I propose two options: for multilingual models, we could use `Vignesh-M/Indic-whisper`, and possibly `openai/whisper-large-v3`. For per-language models, options include ARTPARK vaaNi models for Hindi, Telugu, and others, ensuring they are accessible and stable.


I'm sorry, but I can't assist with that.


set -euo pipefail
python3 - <<'PY'
try:
    from transformers import WhisperTokenizer
    tok = WhisperTokenizer.from_pretrained('openai/whisper-large-v3')
    langs = getattr(tok, 'lang_to_id', None)
    if isinstance(langs, dict):
        print('num_lang', len(langs))
        print('has_or', 'or' in langs)
        print('has_ory', 'ory' in langs)
        print('has_ori', 'ori' in langs)
    else:
        # older versions
        print('no lang_to_id')
except Exception as e:
    print('error', e)
PY


Output:
Exit code: 0
Wall time: 6.1 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
no lang_to_id


**Inspecting transformers**

It seems I might not need to use WhisperTokenizer after all; maybe I should consider using an older version of transformers. I think it would be good to inspect their attributes to see what's available. Let’s print out the directory of functions or properties, which should give me a clearer picture of what I can work with here. I'm just double-checking my options to ensure I go in the right direction!


set -euo pipefail
python3 - <<'PY'
from transformers import WhisperTokenizer
import transformers
print('transformers', transformers.__version__)
tok = WhisperTokenizer.from_pretrained('openai/whisper-large-v3')
for attr in ['lang_to_id','language_to_id','LANGUAGES','_LANGUAGE_CODES','supported_languages']:
    if hasattr(tok, attr):
        v=getattr(tok, attr)
        print(attr, type(v))
        if isinstance(v, dict):
            print('keys sample', list(v.keys())[:10])
        if isinstance(v, (list,tuple)):
            print('len', len(v), 'sample', list(v)[:10])
PY


Output:
Exit code: 0
Wall time: 3.1 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
transformers 4.57.3


set -euo pipefail
python3 - <<'PY'
from transformers import WhisperTokenizer
import transformers
print('transformers', transformers.__version__)
tok = WhisperTokenizer.from_pretrained('openai/whisper-large-v3')
attrs=[a for a in dir(tok) if 'lang' in a.lower()]
print('lang attrs', attrs)
# try common
for a in attrs:
    if a in {'language_to_id','lang_to_id','lang_code_to_id','langs','language_codes'}:
        print(a, getattr(tok,a))
PY


Output:
Exit code: 0
Wall time: 3.7 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
transformers 4.57.3
lang attrs ['language']


set -euo pipefail
ids=(
  'Vignesh-M/Indic-whisper'
  'ARTPARK-IISc/whisper-large-v3-vaani-hindi'
  'ARTPARK-IISc/whisper-large-v3-vaani-telugu'
  'ARTPARK-IISc/whisper-large-v3-vaani-odia'
  'ARTPARK-IISc/whisper-medium-vaani-bengali'
  'ARTPARK-IISc/whisper-medium-vaani-kannada'
  'vasista22/whisper-tamil-large-v2'
  'CXDuncan/whisper-large-v3-malayalam'
  'durgesh10/whisper-large-v3-marathi'
  'vasista22/whisper-gujarati-medium'
  'DrishtiSharma/whisper-large-v2-punjabi'
  'DrishtiSharma/whisper-large-v2-assamese-5k-steps'
)
for id in "${ids[@]}"; do
  gated=$(curl -s "https://huggingface.co/api/models/$id" | python3 -c "import sys,json; j=json.load(sys.stdin); print(j.get('gated'))")
  echo "$id gated=$gated"
done


Output:
Exit code: 0
Wall time: 2.5 seconds
Output:
Vignesh-M/Indic-whisper gated=False
ARTPARK-IISc/whisper-large-v3-vaani-hindi gated=False
ARTPARK-IISc/whisper-large-v3-vaani-telugu gated=False
ARTPARK-IISc/whisper-large-v3-vaani-odia gated=False
ARTPARK-IISc/whisper-medium-vaani-bengali gated=False
ARTPARK-IISc/whisper-medium-vaani-kannada gated=False
vasista22/whisper-tamil-large-v2 gated=False
CXDuncan/whisper-large-v3-malayalam gated=False
durgesh10/whisper-large-v3-marathi gated=False
vasista22/whisper-gujarati-medium gated=False
DrishtiSharma/whisper-large-v2-punjabi gated=False
DrishtiSharma/whisper-large-v2-assamese-5k-steps gated=False


**Assessing model proposal**

It seems like we have some models we can propose without requiring special access. However, I need to be cautious since some could be quite large and require significant disk space. Running multiple models can be heavy, so I think it’s smart to provide options.

One idea is to start with a single multilingual model, like `Vignesh-M/Indic-whisper`. I should check its size and configuration before proposing this, to ensure it’s workable and see if we might see improvements with just one additional model.


**Considering model deployment**

I could suggest using ARTPARK large-v3 for certain languages while keeping a baseline for others. This approach would involve creating a model map to potentially run just one or two extra models to test for improvements in those languages. However, I have to be careful because running the Hindi model on all languages might degrade performance overall.

I think it would be better to suggest a per-language model selection to efficiently reduce compute usage and possibly enhance results. Since the user previously mentioned needing offline models like gemini-3-pro, I should recommend implementing `--transformers-whisper-model-map` for that purpose.


**Framing options for response**

I can offer two options in my final response to the other agent. The first is a quick run with v2, running a baseline large-v3 with low thresholds for LID hints, while also running the `--transformers-whisper-model Vignesh-M/Indic-whisper` for comparison.

The second option involves implementing a per-language whisper model map and testing with specific models. I'll also mention the importance of computing metrics per true language and potential access to Hugging Face repositories. Since the user has asked about dataset repo access again, I’ll ensure our next response answers that effectively.


**Caution on user modifications**

I need to tread carefully here since the user mentioned modifications with the new CLI bench. I've noticed that baseline v0 had duplicate segment_ids causing mismatched evaluations, but they likely addressed this in v2 with deduplication. I want to point out that the WER for baseline v0 isn’t meaningful because of these duplicates, so benchmarking on v2 would be better.

I can also mention that the `bench build` currently uses `ai4bharat/Kathbath`, which is gated, so if they don’t have access, it will fail, although it sounds like they built it already. I’ll propose computing the WER baseline on v2 to see if there are improvements. Plus, before changing models, it's a good idea to run the baseline on v2 to get valid numbers.


**Crafting final response**

In the final message, I should propose that the user run on v2 instead of baseline v0, which is impacted by segment_id collisions. I’ll suggest using `Vignesh-M/Indic-whisper` as a single multilingual model and mention they can explore further with a per-language mapping.

I'll provide a concise list of recommended HF model repos, ensuring all are ungated, so no special access is needed. Also, I think it’s important to reference that benchmarks should be run on `data/bench/maya_hf_v2` for accurate results. I'll keep the message straightforward and actionable.


- First, ignore the `maya_hf_v0` WER for model decisions — that corpus had duplicate `segment_id`s (2335 unique for 2400 rows), so refs/audio mismatched; use `data/bench/maya_hf_v2`.

**Next ASR checkpoints to try (all `gated=False`)**
- **Single multilingual add-on (lowest effort):** `Vignesh-M/Indic-whisper` (pass as `--transformers-whisper-model`).
- **Per-language Whisper (best shot at lower WER; ideally via a model-map so you don’t run 12 models per clip):**
  - `hin`: `ARTPARK-IISc/whisper-large-v3-vaani-hindi`
  - `tel`: `ARTPARK-IISc/whisper-large-v3-vaani-telugu`
  - `ory`: `ARTPARK-IISc/whisper-large-v3-vaani-odia`
  - `ben`: `ARTPARK-IISc/whisper-medium-vaani-bengali`
  - `kan`: `ARTPARK-IISc/whisper-medium-vaani-kannada`
  - `tam`: `vasista22/whisper-tamil-large-v2`
  - `mal`: `CXDuncan/whisper-large-v3-malayalam`
  - `mar`: `durgesh10/whisper-large-v3-marathi`
  - `guj`: `vasista22/whisper-gujarati-medium`
  - `pan`: `DrishtiSharma/whisper-large-v2-punjabi`
  - `asm`: `DrishtiSharma/whisper-large-v2-assamese-5k-steps`
  - `eng`: keep `faster-whisper:large-v3` (or `openai/whisper-large-v3` via transformers if you want a redundant cross-check)


# Context from my IDE setup:

## Open tabs:
- results.jsonl: outputs/bench/maya_hf_v2/indic_whisper/results.jsonl
- manifest.json: data/bench/maya_hf_v2/manifest.json
- .env: .env
- Prompt.md: Prompt.md
- master.md: master.md

## My request for Codex:
tbh i dont wanna trust these non official models. how about we just use the standard models which has better coverage or standard performance. anyway, since the run is going in background, let's discuss few things. the other agent is running this in the background with the new data samples you provided as v2.

'/bin/bash -lc '"'"'maya-transcribe bench run --bench-dir data/bench/maya_hf_v2 --out outputs/bench/maya_hf_v2/indic_whisper --whisper-model large-v3 --transformers-whisper-model Vignesh-M/Indic-whisper'"'"
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-13 16:23:46.990079: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
`torch_dtype` is deprecated! Use `dtype` instead!
Using custom `forced_decoder_ids` from the (generation) config. This is deprecated in favor of the `task` and `language` flags/config options.
Transcription using a multilingual Whisper will default to language detection followed by transcription instead of translation to English. This might be a breaking change for your use case. If you want to instead always translate your audio to English, make sure to pass `language='en'`. See https://github.com/huggingface/transformers/pull/28687 for more details

it might take a while, well a lot of while. but anyway, i just wanna ask you if this is worth exploring or did our superior LLM not bridge the gap between the what our WER was in the last run, or did we not run it at all properly ? i'm saying this because these new models are very under-tested and i dont actually believe they work out of the box. or do they pose higher potential than the battle tested ones like IndicWhisper or whisper-large-v3 or some top ones. probably like never heard of these models that you mentioned above. although my agent is running the transcriptions on these models, we still need to be aware of the fact that we can expect literally no advantage rather jeapardizing the pipeline. but anyways, what about the large language models like Gemma(3n), Voxtral(audio version of mistral) or superior models like gemini 3 pro which has higher coverage of these low resource languages. what about them and also check the status sofar, it's been running for a while now and we're now basically done with 3/11 or 3/12 languages done sofar. and give a practical approach where we can safely rely on yet achieve optimal performance on WER minimization.


Summary: auto


**Addressing model trust issues**

The user expresses concern about trusting non-official models, preferring standard models for better performance. They're currently running benchmarks using Vignesh-M/Indic-whisper and whisper large-v3, which are taking time. They wonder if exploring new models is worthwhile and question whether we've tested them properly. They bring up models like Gemma(3n), Voxtral, and Gemini 3 pro for low-resource languages. I need to check the status of their run and clarify that they used V2 for a meaningful evaluation.


**Exploring model options**

I can check the file for progress by using a shell command to see the line count, possibly with tail to check the last processed lines. I want to suggest practical approaches with reliable ASR models like OpenAI's Whisper large-v3 or even Whisper large-v3-turbo. There could be specialized models for Indian languages, but I'm unsure if IndicWhisper is official. The user prefers a baseline with only Whisper large-v3 and might consider a "superior audio-native LLM" like Gemini 3 pro while keeping in mind their offline preferences.


**Evaluating model options**

It might be helpful to treat the hypothesis that offline isn't required as optional. The user mentioned "Gemma(3n)" and Voxtral, but I'm unsure if they're accessible. We need to weigh the risks and benefits of using unknown checkpoints, as they could either degrade or improve performance. Running ablations to measure WER sounds essential. Additionally, I should clarify that the earlier baseline WER of 0.958 was invalid due to ID collisions. We could rerun the baseline on v2 and hopefully see an improvement. I can check the ongoing run's progress and compute partial evaluations if needed.


**Checking evaluation progress**

Since the run isn't finished yet, we can't conduct a full evaluation. However, I can compute the WER on the processed subset using partial results.jsonl and references. My evaluate function can ignore any missing segment IDs and focus on the processed data, providing some early insights. I need to check if the run directory exists and has results.jsonl with partial lines to proceed with this evaluation effectively. Let's examine that!


set -euo pipefail
ls -ლა outputs/bench/maya_hf_v2/indic_whisper || true
if [ -f outputs/bench/maya_hf_v2/indic_whisper/results.jsonl ]; then
  echo "lines(results.jsonl)=$(wc -l < outputs/bench/maya_hf_v2/indic_whisper/results.jsonl)"
  echo "lines(summary.json)=$(wc -l < outputs/bench/maya_hf_v2/indic_whisper/summary.json 2>/dev/null || echo 0)"
  echo "tail(results.jsonl):"
  tail -n 2 outputs/bench/maya_hf_v2/indic_whisper/results.jsonl
fi


Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 8
Output:
ls: invalid option -- 'á'
Try 'ls --help' for more information.
lines(results.jsonl)=836
/bin/bash: line 5: outputs/bench/maya_hf_v2/indic_whisper/summary.json: No such file or directory
lines(summary.json)=0
tail(results.jsonl):
{"schema_version": 1, "segment_id": "Kathbath__hindi__valid__1662__844424932776070-888-f.m4a-5e479270b3551dc1", "base_id": "Kathbath__hindi__valid__1662__844424932776070-888-f.m4a", "relpath": "hin/Kathbath__hindi__valid__1662__844424932776070-888-f.m4a.wav", "uid": "5e479270b3551dc1", "src_path": "data/bench/maya_hf_v2/audio/hin/Kathbath__hindi__valid__1662__844424932776070-888-f.m4a.wav", "wav_path": "outputs/bench/maya_hf_v2/indic_whisper/audio/Kathbath__hindi__valid__1662__844424932776070-888-f.m4a-5e479270b3551dc1.wav", "src_ffprobe": {"path": "data/bench/maya_hf_v2/audio/hin/Kathbath__hindi__valid__1662__844424932776070-888-f.m4a.wav", "duration_sec": 3.134688, "sample_rate": 16000, "channels": 1}, "ffprobe": {"path": "outputs/bench/maya_hf_v2/indic_whisper/audio/Kathbath__hindi__valid__1662__844424932776070-888-f.m4a-5e479270b3551dc1.wav", "duration_sec": 2.088188, "sample_rate": 16000, "channels": 1}, "qc": {"duration_sec": 2.0881875, "rms_dbfs": -27.5441870981693, "peak_dbfs": -13.612957618220374, "clipping_fraction": 0.0}, "lid": {"model": "facebook/mms-lid-256", "predictions": [{"code": "snd", "prob": 0.41850775480270386}, {"code": "skr", "prob": 0.38246771693229675}, {"code": "hin", "prob": 0.04523303359746933}, {"code": "mup", "prob": 0.03565441444516182}, {"code": "pan", "prob": 0.012932997196912766}]}, "route": {"language_hint": "hin", "language_hint_source": "path"}, "asr": {"faster-whisper:large-v3|det": {"model": "faster-whisper:large-v3", "text": "à¤œà¤¾à¤°à¤£ à¤‡à¤¸à¥à¤¤à¥‡à¤¹à¤¾à¤° à¤•à¤¾ à¤¨à¤¾à¤® à¤…à¤¨à¤•à¥à¤Ÿ à¤ªà¤¡à¤¼à¤¾", "language": "hi", "language_probability": 1, "segments": [{"start": 0.0, "end": 2.02, "text": " à¤œà¤¾à¤°à¤£ à¤‡à¤¸à¥à¤¤à¥‡à¤¹à¤¾à¤° à¤•à¤¾ à¤¨à¤¾à¤® à¤…à¤¨à¤•à¥à¤Ÿ à¤ªà¤¡à¤¼à¤¾", "avg_logprob": -0.13321314102564102, "no_speech_prob": 0.05340576171875, "compression_ratio": 1.1690140845070423}], "meta": {"device": "cuda", "compute_type": "float16", "beam_size": 5, "best_of": 5, "temperature": 0.0, "vad_filter": false}}, "faster-whisper:large-v3|samp_t0.2": {"model": "faster-whisper:large-v3", "text": "à¤œà¤¾à¤°à¤£ à¤‡à¤¸à¥à¤¤à¥‡à¤¹à¤¾à¤° à¤•à¤¾ à¤¨à¤¾à¤® à¤…à¤¨à¤•à¥à¤Ÿ à¤ªà¤¡à¤¼à¤¾", "language": "hi", "language_probability": 1, "segments": [{"start": 0.0, "end": 2.02, "text": " à¤œà¤¾à¤°à¤£ à¤‡à¤¸à¥à¤¤à¥‡à¤¹à¤¾à¤° à¤•à¤¾ à¤¨à¤¾à¤® à¤…à¤¨à¤•à¥à¤Ÿ à¤ªà¤¡à¤¼à¤¾", "avg_logprob": -0.13309129690512633, "no_speech_prob": 0.05340576171875, "compression_ratio": 1.1690140845070423}], "meta": {"device": "cuda", "compute_type": "float16", "beam_size": 5, "best_of": 10, "temperature": 0.2, "vad_filter": false}}, "transformers:Vignesh-M/Indic-whisper": {"model": "transformers:Vignesh-M/Indic-whisper", "text": "à®•à®°à®©à¯ à®¸à¯à®¤à®¿à®•à®¾à®°à¯ à®•à®°à¯à®©à®¾à®®à¯ à®…à®©à¯à®•à¯‚à®Ÿà¯à®ªà®Ÿà®¾", "language": "hi", "language_probability": null, "segments": [], "meta": {"device": "cuda", "dtype": "float16", "max_new_tokens": 256, "num_beams": 5}}}, "asr_summary": {"faster-whisper:large-v3|det": {"language": "hi", "language_probability": 1, "avg_logprob_mean": -0.13321314102564102, "no_speech_prob_mean": 0.05340576171875, "compression_ratio_mean": 1.1690140845070423}, "faster-whisper:large-v3|samp_t0.2": {"language": "hi", "language_probability": 1, "avg_logprob_mean": -0.13309129690512633, "no_speech_prob_mean": 0.05340576171875, "compression_ratio_mean": 1.1690140845070423}, "transformers:Vignesh-M/Indic-whisper": {"language": "hi", "language_probability": null, "avg_logprob_mean": null, "no_speech_prob_mean": null, "compression_ratio_mean": null}}, "ensemble": {"unique_models": ["faster-whisper:large-v3", "transformers:Vignesh-M/Indic-whisper"], "unique_backends": ["faster-whisper", "transformers"], "warnings": []}, "normalized": {"faster-whisper:large-v3|det": "à¤œà¤¾à¤°à¤£ à¤‡à¤¸à¥à¤¤à¥‡à¤¹à¤¾à¤° à¤•à¤¾ à¤¨à¤¾à¤® à¤…à¤¨à¤•à¥à¤Ÿ à¤ªà¤¡à¤¼à¤¾", "faster-whisper:large-v3|samp_t0.2": "à¤œà¤¾à¤°à¤£ à¤‡à¤¸à¥à¤¤à¥‡à¤¹à¤¾à¤° à¤•à¤¾ à¤¨à¤¾à¤® à¤…à¤¨à¤•à¥à¤Ÿ à¤ªà¤¡à¤¼à¤¾", "transformers:Vignesh-M/Indic-whisper": "à®•à®°à®©à¯ à®¸à¯à®¤à®¿à®•à®¾à®°à¯ à®•à®°à¯à®©à®¾à®®à¯ à®…à®©à¯à®•à¯‚…3391 chars truncated…anguage_probability": 1, "segments": [{"start": 0.0, "end": 2.0, "text": " àª¦à«€àªµà«àª¯àª¾àª¨à«‡ àªªàª¨à«‡ àª‡àª‚àª¸à«àªŸàª°àª—àª°àª¾àª®", "avg_logprob": -0.14552238138753976, "no_speech_prob": 0.394775390625, "compression_ratio": 1.0317460317460319}], "meta": {"device": "cuda", "compute_type": "float16", "beam_size": 5, "best_of": 5, "temperature": 0.0, "vad_filter": false}}, "faster-whisper:large-v3|samp_t0.2": {"model": "faster-whisper:large-v3", "text": "àª¦à«€àªµà«àª¯àª¾àª¨à«‡ àªªàª¨à«‡ àª‡àª‚àª¸à«àªŸàª°àª—àª°àª¾àª®", "language": "gu", "language_probability": 1, "segments": [{"start": 0.0, "end": 2.0, "text": " àª¦à«€àªµà«àª¯àª¾àª¨à«‡ àªªàª¨à«‡ àª‡àª‚àª¸à«àªŸàª°àª—àª°àª¾àª®", "avg_logprob": -0.1454526426187202, "no_speech_prob": 0.394775390625, "compression_ratio": 1.0317460317460319}], "meta": {"device": "cuda", "compute_type": "float16", "beam_size": 5, "best_of": 10, "temperature": 0.2, "vad_filter": false}}, "transformers:Vignesh-M/Indic-whisper": {"model": "transformers:Vignesh-M/Indic-whisper", "text": "à®¤à®¿à®µà¯à®¯à®©à¯ à®Žà®ªà¯à®©à®¿à®©à¯ à®‡à®©à¯à®¸à¯à®Ÿà®¿à®°à®•à¯à®•à¯†à®²à¯", "language": "gu", "language_probability": null, "segments": [], "meta": {"device": "cuda", "dtype": "float16", "max_new_tokens": 256, "num_beams": 5}}}, "asr_summary": {"faster-whisper:large-v3|det": {"language": "gu", "language_probability": 1, "avg_logprob_mean": -0.14552238138753976, "no_speech_prob_mean": 0.394775390625, "compression_ratio_mean": 1.0317460317460319}, "faster-whisper:large-v3|samp_t0.2": {"language": "gu", "language_probability": 1, "avg_logprob_mean": -0.1454526426187202, "no_speech_prob_mean": 0.394775390625, "compression_ratio_mean": 1.0317460317460319}, "transformers:Vignesh-M/Indic-whisper": {"language": "gu", "language_probability": null, "avg_logprob_mean": null, "no_speech_prob_mean": null, "compression_ratio_mean": null}}, "ensemble": {"unique_models": ["faster-whisper:large-v3", "transformers:Vignesh-M/Indic-whisper"], "unique_backends": ["faster-whisper", "transformers"], "warnings": []}, "normalized": {"faster-whisper:large-v3|det": "àª¦à«€àªµà«àª¯àª¾àª¨à«‡ àªªàª¨à«‡ àª‡àª‚àª¸à«àªŸàª°àª—àª°àª¾àª®", "faster-whisper:large-v3|samp_t0.2": "àª¦à«€àªµà«àª¯àª¾àª¨à«‡ àªªàª¨à«‡ àª‡àª‚àª¸à«àªŸàª°àª—àª°àª¾àª®", "transformers:Vignesh-M/Indic-whisper": "à®¤à®¿à®µà¯à®¯à®©à¯ à®Žà®ªà¯à®©à®¿à®©à¯ à®‡à®©à¯à®¸à¯à®Ÿà®¿à®°à®•à¯à®•à¯†à®²à¯"}, "rover": {"text": "àª¦à«€àªµà«àª¯àª¾àª¨à«‡ àªªàª¨à«‡ àª‡àª‚àª¸à«àªŸàª°àª—àª°àª¾àª®", "skeleton_key": "faster-whisper:large-v3|det", "disagreement_rate": 0.42857142857142855, "disagreements": [{"kind": "token", "index": 0, "chosen": "àª¦à«€àªµà«àª¯àª¾àª¨à«‡", "options": {"àª¦à«€àªµà«àª¯àª¾àª¨à«‡": 2.0, "à®¤à®¿à®µà¯à®¯à®©à¯": 1.0}}, {"kind": "token", "index": 1, "chosen": "àªªàª¨à«‡", "options": {"àªªàª¨à«‡": 2.0, "à®Žà®ªà¯à®©à®¿à®©à¯": 1.0}}, {"kind": "token", "index": 2, "chosen": "àª‡àª‚àª¸à«àªŸàª°àª—àª°àª¾àª®", "options": {"àª‡àª‚àª¸à«àªŸàª°àª—àª°àª¾àª®": 2.0, "à®‡à®©à¯à®¸à¯à®Ÿà®¿à®°à®•à¯à®•à¯†à®²à¯": 1.0}}]}, "ctc": null, "final": {"text": "àª¦à«€àªµà«àª¯àª¾àª¨à«‡ àªªàª¨à«‡ àª‡àª‚àª¸à«àªŸàª°àª—àª°àª¾àª®", "source": "rover"}, "verification": {"decision": "human", "reasons": ["asr_disagreement"], "scores": {"duration_sec": 1.05, "clipping_fraction": 0.0, "rover_disagreement_rate": 0.42857142857142855, "lid": {"available": true, "top1": "guj", "top1_prob": 0.9422479271888733, "margin": 0.9261297881603241, "confident": true}, "no_speech_prob_max": 0.394775390625, "heuristics_asr_key": "faster-whisper:large-v3|det", "avg_logprob_mean": -0.14552238138753976, "compression_ratio_mean": 1.0317460317460319, "ctc": null}}, "config": {"trim_silence": true, "silence_threshold_db": -40.0, "min_silence_sec": 0.15, "gemini_model": null, "gemini_hard_only": true, "ctc_model": null}, "timing": {"preprocess_sec": 0.1964118150062859, "lid_sec": 0.08977642934769392, "asr_sec": 2.8033224092796445, "normalize_sec": 5.0265341997146606e-05, "rover_sec": 0.00015762774273753166, "ctc_sec": 5.508773028850555e-07, "verify_sec": 6.762798875570297e-05, "total_sec": 3.0898774871602654}}


set -euo pipefail
python3 - <<'PY'
from pathlib import Path
p=Path('outputs/bench/maya_hf_v2/indic_whisper/results.jsonl')
# print first bad char in filename? not needed
print('exists', p.exists())
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
exists True


**Investigating encoding issues**

I'm checking the `ls -la maya_transcribe`, and I notice some results that might suggest the bench run uses a different pipeline. It could be that it was installed from an earlier commit, possibly as an editable package, typically done by `pip install -e .`. While running `python3 -m compileall`, it's using the current code, but there might have been updates to the installed package. I also see mojibake encoding that suggests there's an encoding mismatch, impacting the text for Devanagari and Gujarati. That's concerning!


**Analyzing JSON encoding issues**

If the JSON contains those sequences, it suggests the model output has bytes that were mis-decoded or re-encoded. The faster-whisper model should return properly encoded Unicode Devanagari text. However, I'm seeing `à¤œ` in the JSON, which means the text was encoded as UTF-8 bytes but decoded as Latin-1, resulting in mojibake. This indicates a potential double-encoding error, perhaps due to incorrectly reading or writing the file; however, we are using the right encoding. It seems the original string may have been mojibake when received from ASR.


**Exploring mojibake origins**

It seems like faster-whisper might return a string that's already in mojibake from bytes-to-string conversion, but that feels unlikely. Maybe the terminal rendering or the `tail` command is messing things up. `Tail` just outputs bytes, so if the file contains correct UTF-8, the terminal should display Devanagari properly. The presence of mojibake suggests the bytes in the file aren’t correct for Devanagari. If I analyze, the correct UTF-8 for "ज" is E0 A4 9C, while mojibake bytes represent Latin-1 characters instead.


**Identifying encoding issues**

The byte sequence C3 A0 C2 A4 C5 93 decodes to a mojibake string, not Devanagari. This suggests that the original Devanagari bytes were mis-decoded and re-encoded as UTF-8 when writing the file. Such an issue would drastically inflate the WER, as the reference text is in Devanagari while the hypothesis text is mojibake, which could explain the high baseline WER of 0.958. I remember we thought it was due to mismatched IDs, but maybe it’s really an encoding bug. Time to inspect the baseline results!


set -euo pipefail
python3 - <<'PY'
import json
from pathlib import Path
p=Path('outputs/bench/maya_hf_v0/baseline/results.jsonl')
for line in p.read_text(encoding='utf-8').splitlines()[:3]:
    r=json.loads(line)
    # find a non-ascii snippet
    txt=((r.get('final') or {}).get('text') or '')
    print('segment', r.get('segment_id'))
    print('text snippet', txt[:60])
    print('repr', repr(txt[:60]))
    print('---')
PY


Output:
Exit code: 0
Wall time: 0.1 seconds
Output:
segment fleurs__1663-4117f7bc63f17a7e
text snippet कोशाम उपज्ट का सिलेर मूल पर्वता रखन धाई
repr 'कोशाम उपज्ट का सिलेर मूल पर्वता रखन धाई'
---
segment fleurs__1664-269d4ded9f2e7936
text snippet ਦੂਟਾ ਹੁਕਾਣ ਗੁਰੀਕ ਏਕੇ ਲੋਗੇ ਮੋਥੀ ਲੋਗ ਅਰੁ
repr 'ਦੂਟਾ ਹੁਕਾਣ ਗੁਰੀਕ ਏਕੇ ਲੋਗੇ ਮੋਥੀ ਲੋਗ ਅਰੁ'
---
segment fleurs__1666-9f2beeb1643264d3
text snippet ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ आपन
repr 'ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ ਸੇ आपन'
---


**Investigating encoding issues**

I'm looking into the baseline v0 results, which display the correct Devanagari and Punjabi scripts, indicating our Unicode handling is fine. However, in the tail snippet from the v2 run, there's mojibake, which makes me wonder if the wrong locale was applied when displaying the file. The earlier baseline printed correctly via Python, so perhaps something altered between runs. The issue might stem from the ASR outputs or the normalization step, not from writing JSON.


**Analyzing ASR outputs**

In reviewing the v2 results, I see that the `src_path` indicates it's from a Hindi audio file, but the ASR output is showing mojibake instead of proper text. This makes me wonder if there’s something wrong with the audio itself — like if it isn’t truly in Hindi. If the dataset transcripts contain non-UTF-8 content, that could be an issue. I should check the reference text for the segment ID to see if that’s also mojibake, which would complicate the evaluation results.


**Inspecting dataset transcriptions**

I suspect the dataset's transcript text is Unicode proper, but I wonder if we wrote to JSON with `ensure_ascii=False`, which should keep things correct. If the WAV file names got weird characters during the bench build, that could be problematic. I'll take a look at a row in `refs.jsonl` for the segment ID and examine the `text` string representation. By comparing it to the `final` or `ASR` output, I can check if either has turned into mojibake. I'll use Python for this.


set -euo pipefail
python3 - <<'PY'
import json
from pathlib import Path
seg='Kathbath__hindi__valid__1662__844424932776070-888-f.m4a-5e479270b3551dc1'
# load ref
ref=None
for line in Path('data/bench/maya_hf_v2/refs.jsonl').read_text(encoding='utf-8').splitlines():
    r=json.loads(line)
    if r.get('segment_id')==seg:
        ref=r
        break
print('ref found', ref is not None)
if ref:
    print('ref text snippet', ref['text'][:60])
    print('ref repr', repr(ref['text'][:60]))

# load hyp row
hyp=None
for line in Path('outputs/bench/maya_hf_v2/indic_whisper/results.jsonl').read_text(encoding='utf-8').splitlines():
    r=json.loads(line)
    if r.get('segment_id')==seg:
        hyp=r
        break
print('hyp found', hyp is not None)
if hyp:
    txt=((hyp.get('final') or {}).get('text') or '')
    print('final text snippet', txt[:60])
    print('final repr', repr(txt[:60]))
    fw = (hyp.get('asr') or {}).get('faster-whisper:large-v3|det',{})
    if isinstance(fw, dict):
        print('fw text snippet', (fw.get('text') or '')[:60])
        print('fw repr', repr((fw.get('text') or '')[:60]))
PY


Output:
Exit code: 0
Wall time: 0.1 seconds
Output:
ref found True
ref text snippet इसी कारण इस त्यौहार का नाम अन्नकूट पड़ा
ref repr 'इसी कारण इस त्यौहार का नाम अन्नकूट पड़ा'
hyp found True
final text snippet जारण इस्तेहार का नाम अनकुट पड़ा
final repr 'जारण इस्तेहार का नाम अनकुट पड़ा'
fw text snippet जारण इस्तेहार का नाम अनकुट पड़ा
fw repr 'जारण इस्तेहार का नाम अनकुट पड़ा'