Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/usr/bin/env python3
"""
Evaluation script using IndicASR (ai4bharat/indic-conformer-600m-multilingual) + Gemini pipeline.

IndicConformer is AI4Bharat's state-of-the-art ASR model for all 22 Indian languages,
achieving significantly better WER than vanilla Whisper on Indic languages.

Run with: python3 -m transcribe.eval_indicwhisper
"""

import json
import time
import logging
import tarfile
import argparse
from pathlib import Path
from dataclasses import dataclass, asdict
from typing import List, Dict, Optional
import numpy as np

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

@dataclass
class SampleResult:
    sample_id: str
    language: str
    reference: str
    indicwhisper_hyp: str
    gemini_hyp: str
    consensus_hyp: str
    indicwhisper_wer: float
    gemini_wer: float
    consensus_wer: float
    indicwhisper_cer: float
    gemini_cer: float
    consensus_cer: float
    agreement_category: str
    processing_time: float

def load_audio_from_tar(tar_path: Path, audio_name: str) -> tuple:
    """Load audio from tar file."""
    import soundfile as sf
    import io

    with tarfile.open(tar_path, "r") as tar:
        audio_member = tar.getmember(audio_name)
        audio_f = tar.extractfile(audio_member)
        audio_bytes = audio_f.read()
        audio, sr = sf.read(io.BytesIO(audio_bytes))

        if audio.dtype != np.float32:
            audio = audio.astype(np.float32)
        if len(audio.shape) > 1:
            audio = audio.mean(axis=1)

        return audio, sr

def get_samples_from_tar(tar_path: Path, max_samples: int = 10) -> List[dict]:
    """Get samples from tar."""
    samples = []
    with tarfile.open(tar_path, "r") as tar:
        json_members = [m for m in tar.getmembers() if m.name.endswith(".json")]
        for member in json_members[:max_samples]:
            f = tar.extractfile(member)
            if f:
                data = json.load(f)
                data["audio_file"] = member.name.replace(".json", ".wav")
                samples.append(data)
    return samples

def run_evaluation(
    languages: List[str],
    samples_per_lang: int = 10,
    output_path: Optional[Path] = None,
):
    """Run IndicWhisper + Gemini pipeline evaluation."""
    from transcribe.config import config
    from transcribe.pipeline import IndicWhisperPipeline
    from transcribe.utils.text_processing import compute_wer, compute_cer, normalize_for_wer

    eval_dir = config.eval_dir

    # Initialize pipeline
    logger.info("Loading IndicWhisper + Gemini pipeline...")
    pipeline = IndicWhisperPipeline(config)
    pipeline.load()
    logger.info("Pipeline loaded successfully!")

    results_by_lang: Dict[str, List[SampleResult]] = {}

    for language in languages:
        tar_path = eval_dir / f"{language}_eval.tar"
        if not tar_path.exists():
            logger.warning(f"Skipping {language} - tar not found at {tar_path}")
            continue

        logger.info(f"\n{'='*60}")
        logger.info(f"EVALUATING {language.upper()}")
        logger.info("=" * 60)

        samples = get_samples_from_tar(tar_path, samples_per_lang)
        results = []

        for i, sample in enumerate(samples):
            sample_id = sample.get("id", f"{language}_{i}")
            reference = sample.get("transcription", "")
            audio_file = sample.get("audio_file")

            logger.info(f"\n[{i+1}/{len(samples)}] {sample_id}")

            try:
                # Load audio
                audio, sr = load_audio_from_tar(tar_path, audio_file)

                # Run pipeline
                start = time.time()
                result = pipeline.transcribe(audio, sr, language)
                elapsed = time.time() - start

                # Normalize for WER computation
                ref_norm = normalize_for_wer(reference, language)
                indicwhisper_norm = normalize_for_wer(result.whisper_text, language)  # stored in whisper_text field
                gemini_norm = normalize_for_wer(result.gemini_text, language)
                consensus_norm = normalize_for_wer(result.text, language)

                # Compute WER
                indicwhisper_wer = compute_wer(ref_norm, indicwhisper_norm)["wer"]
                gemini_wer = compute_wer(ref_norm, gemini_norm)["wer"]
                consensus_wer = compute_wer(ref_norm, consensus_norm)["wer"]

                # Compute CER
                indicwhisper_cer = compute_cer(ref_norm, indicwhisper_norm)["cer"]
                gemini_cer = compute_cer(ref_norm, gemini_norm)["cer"]
                consensus_cer = compute_cer(ref_norm, consensus_norm)["cer"]

                sample_result = SampleResult(
                    sample_id=sample_id,
                    language=language,
                    reference=reference,
                    indicwhisper_hyp=result.whisper_text,
                    gemini_hyp=result.gemini_text,
                    consensus_hyp=result.text,
                    indicwhisper_wer=indicwhisper_wer,
                    gemini_wer=gemini_wer,
                    consensus_wer=consensus_wer,
                    indicwhisper_cer=indicwhisper_cer,
                    gemini_cer=gemini_cer,
                    consensus_cer=consensus_cer,
                    agreement_category=result.agreement_category.value,
                    processing_time=elapsed,
                )
                results.append(sample_result)

                # Log results
                logger.info(f"  IndicWhisper WER: {indicwhisper_wer*100:.1f}% | Gemini WER: {gemini_wer*100:.1f}% | Consensus WER: {consensus_wer*100:.1f}%")
                logger.info(f"  Category: {result.agreement_category.value}")
                logger.info(f"  REF: {reference[:60]}...")
                logger.info(f"  IW:  {result.whisper_text[:60]}...")
                logger.info(f"  GEM: {result.gemini_text[:60]}...")

            except Exception as e:
                logger.error(f"  Error: {e}")
                import traceback
                traceback.print_exc()

        results_by_lang[language] = results

        # Language summary
        if results:
            avg_iw_wer = np.mean([r.indicwhisper_wer for r in results])
            avg_gem_wer = np.mean([r.gemini_wer for r in results])
            avg_cons_wer = np.mean([r.consensus_wer for r in results])
            logger.info(f"\n{language.upper()} SUMMARY:")
            logger.info(f"  IndicWhisper: WER={avg_iw_wer*100:.2f}%")
            logger.info(f"  Gemini:       WER={avg_gem_wer*100:.2f}%")
            logger.info(f"  Consensus:    WER={avg_cons_wer*100:.2f}%")

    # Print final summary
    print_summary(results_by_lang)

    # Save results
    if output_path:
        save_results(results_by_lang, output_path)

    # Cleanup
    pipeline.unload()

    return results_by_lang

def print_summary(results_by_lang: Dict[str, List[SampleResult]]):
    """Print summary table."""
    print("\n" + "=" * 90)
    print("INDICWHISPER + GEMINI PIPELINE EVALUATION RESULTS")
    print("=" * 90)

    print(f"\n{'Language':<12} {'Samples':>8} {'IW WER':>10} {'Gem WER':>10} {'Cons WER':>10} {'IW CER':>10} {'Gem CER':>10}")
    print("-" * 80)

    all_results = []
    for lang, results in sorted(results_by_lang.items()):
        if not results:
            continue

        iw_wers = [r.indicwhisper_wer for r in results]
        gem_wers = [r.gemini_wer for r in results]
        cons_wers = [r.consensus_wer for r in results]
        iw_cers = [r.indicwhisper_cer for r in results]
        gem_cers = [r.gemini_cer for r in results]

        print(f"{lang:<12} {len(results):>8} {np.mean(iw_wers)*100:>9.2f}% {np.mean(gem_wers)*100:>9.2f}% {np.mean(cons_wers)*100:>9.2f}% {np.mean(iw_cers)*100:>9.2f}% {np.mean(gem_cers)*100:>9.2f}%")
        all_results.extend(results)

    if all_results:
        print("-" * 80)
        all_iw_wers = [r.indicwhisper_wer for r in all_results]
        all_gem_wers = [r.gemini_wer for r in all_results]
        all_cons_wers = [r.consensus_wer for r in all_results]
        all_iw_cers = [r.indicwhisper_cer for r in all_results]
        all_gem_cers = [r.gemini_cer for r in all_results]

        print(f"{'OVERALL':<12} {len(all_results):>8} {np.mean(all_iw_wers)*100:>9.2f}% {np.mean(all_gem_wers)*100:>9.2f}% {np.mean(all_cons_wers)*100:>9.2f}% {np.mean(all_iw_cers)*100:>9.2f}% {np.mean(all_gem_cers)*100:>9.2f}%")

        # Comparison vs vanilla Whisper (from previous evaluation)
        print("\n" + "-" * 50)
        print("COMPARISON (vs vanilla Whisper from previous eval):")
        print("-" * 50)
        print("  Telugu: Whisper was 57.62% WER, IndicWhisper should be much lower")
        print("  Hindi:  Whisper was 26.63% WER, IndicWhisper should be much lower")
        print("  English: Whisper was 0.32% WER (already good)")

        # Agreement categories


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Clarifying normalization in WER computation**

The evaluation calls manual normalization before compute_wer, which itself re-normalizes without removing punctuation, but that doesn't undo prior punctuation removal; thus, WER operates on consistent normalized text despite dual normalization. However, stored hypothesis strings in results still include punctuation, suggesting original texts are saved but WER compares normalized versions.


**Identifying tokenization impact on WER**

WER inflation seems driven by tokenization differences around language-specific postpositions and spacing, especially in Telugu where suffixes may split or merge differently. Normalizing such spacing could improve WER consistency across hypotheses and references. Next, inspecting the IndicWhisper pipeline may clarify differences from WeightedROVER.


sed -n '520,820p' transcribe/pipeline.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:

        self._loaded = True

    def transcribe(
        self,
        audio: np.ndarray,
        sample_rate: int,
        language: Optional[str] = None,
    ) -> PipelineResult:
        """Transcribe using Whisper + Gemini consensus."""
        if not self._loaded:
            self.load()

        start_time = time.time()

        # Get both transcripts
        whisper_result = self.whisper.transcribe(audio, sample_rate, language)
        gemini_result = self.gemini.transcribe(audio, sample_rate, language)

        # Simple comparison
        whisper_norm = normalize_text(whisper_result.text, lowercase=True)
        gemini_norm = normalize_text(gemini_result.text, lowercase=True)

        if whisper_norm == gemini_norm:
            # Agreement
            final_text = gemini_result.text  # prefer Gemini for formatting
            confidence = 0.95
            category = TriageCategory.ALL_AGREE
        else:
            # Prefer Gemini for Indic languages, Whisper for English
            if language and language not in ["en", "english"]:
                final_text = gemini_result.text
                confidence = gemini_result.confidence
            else:
                # Use Gemini's structured output
                final_text = gemini_result.text
                confidence = (gemini_result.confidence + whisper_result.confidence) / 2
            category = TriageCategory.DISPUTED

        return PipelineResult(
            text=final_text,
            text_native=gemini_result.text_native,
            confidence=confidence,
            alignment_score=confidence,
            agreement_category=category,
            whisper_text=whisper_result.text,
            gemini_text=gemini_result.text,
            language=gemini_result.language,
            languages_present=gemini_result.languages_present,
            contains_code_mixing=gemini_result.contains_code_mixing,
            processing_time=time.time() - start_time,
            audio_duration=len(audio) / sample_rate,
        )

class IndicWhisperPipeline:
    """
    Pipeline using IndicWhisper + Gemini for Indic languages.

    IndicWhisper is AI4Bharat's fine-tuned Whisper model specifically for
    Indian languages, achieving significantly lower WER than vanilla Whisper.
    """

    def __init__(self, config: Optional[Config] = None):
        self.config = config or default_config
        self.indicwhisper: Optional[IndicWhisperMultiTranscriber] = None
        self.gemini: Optional[GeminiTranscriber] = None
        self.aligner: Optional[MMSForcedAligner] = None

        # ROVER with language-specific weights
        self.rover = WeightedROVER(
            base_weights=self.config.pipeline.language_weights
        )
        self._loaded = False

    def load(self) -> None:
        """Load all models."""
        if self._loaded:
            return

        logger.info("Loading IndicWhisper pipeline models...")

        self.indicwhisper = IndicWhisperMultiTranscriber(
            device=self.config.model.whisper_device,
        )
        self.indicwhisper.load()
        logger.info("IndicWhisper loaded")

        self.gemini = GeminiTranscriber(
            api_key=self.config.model.gemini_api_key,
            model_name=self.config.model.gemini_model,
            temperature=self.config.model.gemini_temperature,
        )
        self.gemini.load()
        logger.info("Gemini loaded")

        # Load aligner for disputed cases
        self.aligner = MMSForcedAligner(device=self.config.model.whisper_device)
        self.aligner.load()
        logger.info("MMS Aligner loaded")

        self._loaded = True

    def unload(self) -> None:
        """Unload all models."""
        if self.indicwhisper:
            self.indicwhisper.unload()
        if self.gemini:
            self.gemini.unload()
        if self.aligner:
            self.aligner.unload()
        self._loaded = False

    def transcribe(
        self,
        audio: np.ndarray,
        sample_rate: int,
        language: Optional[str] = None,
    ) -> PipelineResult:
        """
        Transcribe using IndicWhisper + Gemini consensus.

        For Indic languages, IndicWhisper provides much better WER than
        vanilla Whisper. Combined with Gemini for consensus.
        """
        if not self._loaded:
            self.load()

        start_time = time.time()
        audio_duration = len(audio) / sample_rate

        # Get transcripts from both models
        transcripts = {}
        transcript_results = {}

        try:
            indicwhisper_result = self.indicwhisper.transcribe(audio, sample_rate, language)
            transcripts["indicwhisper"] = indicwhisper_result.text
            transcript_results["indicwhisper"] = indicwhisper_result
        except Exception as e:
            logger.warning(f"IndicWhisper failed: {e}")

        try:
            gemini_result = self.gemini.transcribe(audio, sample_rate, language)
            transcripts["gemini"] = gemini_result.text
            transcript_results["gemini"] = gemini_result
        except Exception as e:
            logger.warning(f"Gemini failed: {e}")

        # Normalize for comparison
        indicwhisper_norm = normalize_text(transcripts.get("indicwhisper", ""), lowercase=True, remove_punctuation=True)
        gemini_norm = normalize_text(transcripts.get("gemini", ""), lowercase=True, remove_punctuation=True)

        if indicwhisper_norm == gemini_norm:
            # Agreement - use Gemini's formatting
            final_text = transcripts.get("gemini", transcripts.get("indicwhisper", ""))
            confidence = 0.95
            category = TriageCategory.ALL_AGREE
            alignment_score = 0.95
        elif not transcripts.get("indicwhisper"):
            # IndicWhisper failed, use Gemini
            final_text = transcripts.get("gemini", "")
            confidence = transcript_results.get("gemini", TranscriptionResult("", "")).confidence
            category = TriageCategory.SINGLE
            alignment_score = confidence
        elif not transcripts.get("gemini"):
            # Gemini failed, use IndicWhisper
            final_text = transcripts.get("indicwhisper", "")
            confidence = 0.85
            category = TriageCategory.SINGLE
            alignment_score = 0.85
        else:
            # Disagreement - use alignment to decide
            final_text, confidence, alignment_score = self._resolve_disputed(
                audio, sample_rate, transcripts, language
            )
            category = TriageCategory.DISPUTED

        # Get native script version
        text_native = None
        if "gemini" in transcript_results:
            text_native = transcript_results["gemini"].text_native

        # Determine metadata
        detected_language = language or "unknown"
        languages_present = []
        contains_code_mixing = False

        if "gemini" in transcript_results:
            gemini_result = transcript_results["gemini"]
            detected_language = gemini_result.language
            languages_present = gemini_result.languages_present
            contains_code_mixing = gemini_result.contains_code_mixing

        processing_time = time.time() - start_time

        return PipelineResult(
            text=final_text,
            text_native=text_native,
            confidence=confidence,
            alignment_score=alignment_score,
            agreement_category=category,
            whisper_text=transcripts.get("indicwhisper", ""),  # Store in whisper_text field
            gemini_text=transcripts.get("gemini", ""),
            language=detected_language,
            languages_present=languages_present,
            contains_code_mixing=contains_code_mixing,
            processing_time=processing_time,
            audio_duration=audio_duration,
            needs_review=confidence < self.config.pipeline.high_confidence_threshold,
            review_reason=f"Confidence {confidence:.2f}" if confidence < self.config.pipeline.high_confidence_threshold else "",
        )

    def _resolve_disputed(
        self,
        audio: np.ndarray,
        sample_rate: int,
        transcripts: Dict[str, str],
        language: Optional[str],
    ) -> Tuple[str, float, float]:
        """
        Resolve disagreement using forced alignment scoring.

        Returns:
            Tuple of (final_text, confidence, alignment_score)
        """
        alignment_scores = {}

        for model_name, text in transcripts.items():
            if not text:
                continue
            try:
                result = self.aligner.align(audio, sample_rate, text, language)
                alignment_scores[model_name] = result.overall_score
            except Exception as e:
                logger.warning(f"Alignment failed for {model_name}: {e}")
                alignment_scores[model_name] = 0.5  # Default score

        if not alignment_scores:
            # Fallback to Gemini if alignment fails
            return transcripts.get("gemini", transcripts.get("indicwhisper", "")), 0.7, 0.7

        # Pick the transcript with best alignment score
        best_model = max(alignment_scores, key=alignment_scores.get)
        best_score = alignment_scores[best_model]

        return transcripts[best_model], best_score, best_score


Summary: auto


**Investigating alignment scoring bias**

Noticed alignment scoring may favor spelled-out digits due to aligner vocabulary, causing consensus to prefer IndicWhisper even when Gemini yields lower WER. Next, I'll inspect the aligner implementation to understand token handling and potential bias.


sed -n '1,240p' transcribe/alignment/aligner.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""Base forced alignment classes."""

from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import List, Optional, Tuple
import numpy as np

@dataclass
class AlignedWord:
    """A word with alignment information."""
    word: str
    start: float  # seconds
    end: float    # seconds
    score: float  # alignment confidence/score
    label: Optional[str] = None  # phoneme label if available

@dataclass
class AlignmentResult:
    """Result from forced alignment."""
    words: List[AlignedWord]
    overall_score: float
    coverage: float  # percentage of words successfully aligned
    duration: float

    # Detailed metrics
    avg_word_score: float = 0.0
    min_word_score: float = 0.0
    unaligned_words: List[str] = field(default_factory=list)

    # Timing sanity metrics
    timing_issues: int = 0  # count of timing anomalies

    def to_dict(self) -> dict:
        return {
            "words": [
                {"word": w.word, "start": w.start, "end": w.end, "score": w.score}
                for w in self.words
            ],
            "overall_score": self.overall_score,
            "coverage": self.coverage,
            "duration": self.duration,
            "avg_word_score": self.avg_word_score,
            "min_word_score": self.min_word_score,
            "unaligned_words": self.unaligned_words,
            "timing_issues": self.timing_issues,
        }

class ForcedAligner(ABC):
    """Base class for forced aligners."""

    def __init__(self, language: str = "hi"):
        self.language = language
        self._loaded = False

    @abstractmethod
    def load(self) -> None:
        """Load the alignment model."""
        pass

    @abstractmethod
    def align(
        self,
        audio: np.ndarray,
        sample_rate: int,
        transcript: str,
        language: Optional[str] = None,
    ) -> AlignmentResult:
        """
        Align transcript to audio.

        Args:
            audio: Audio array
            sample_rate: Sample rate
            transcript: Text transcript to align
            language: Optional language code

        Returns:
            AlignmentResult with word-level alignments and scores
        """
        pass

    def score_transcript(
        self,
        audio: np.ndarray,
        sample_rate: int,
        transcript: str,
        language: Optional[str] = None,
    ) -> Tuple[float, float]:
        """
        Score how well a transcript matches audio.

        Returns:
            Tuple of (alignment_score, coverage)
        """
        result = self.align(audio, sample_rate, transcript, language)
        return result.overall_score, result.coverage

    def validate_timing(self, words: List[AlignedWord]) -> int:
        """
        Check for timing anomalies.

        Returns:
            Count of timing issues found
        """
        issues = 0

        for i, word in enumerate(words):
            # Check for negative duration
            if word.end <= word.start:
                issues += 1
                continue

            # Check for unrealistically short words (< 50ms)
            duration = word.end - word.start
            if duration < 0.05 and len(word.word) > 2:
                issues += 1

            # Check for unrealistically long words (> 3s for single word)
            if duration > 3.0:
                issues += 1

            # Check for gaps between words (> 2s without pause marker)
            if i > 0:
                gap = word.start - words[i-1].end
                if gap > 2.0:
                    issues += 1

        return issues

    def unload(self) -> None:
        """Unload model from memory."""
        self._loaded = False

    @property
    def is_loaded(self) -> bool:
        return self._loaded


**Starting mms_aligner review**


sed -n '1,260p' transcribe/alignment/mms_aligner.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""MMS (Massively Multilingual Speech) Forced Aligner."""

import time
from typing import Optional, List
import numpy as np

from .aligner import ForcedAligner, AlignmentResult, AlignedWord

# ISO 639-3 codes for MMS
MMS_LANGUAGE_MAP = {
    "hi": "hin",   # Hindi
    "te": "tel",   # Telugu
    "ta": "tam",   # Tamil
    "kn": "kan",   # Kannada
    "ml": "mal",   # Malayalam
    "bn": "ben",   # Bengali
    "gu": "guj",   # Gujarati
    "mr": "mar",   # Marathi
    "pa": "pan",   # Punjabi
    "or": "ori",   # Odia
    "as": "asm",   # Assamese
    "en": "eng",   # English
}

class MMSForcedAligner(ForcedAligner):
    """
    Forced aligner using Facebook's MMS model.

    Uses CTC-based alignment for 1000+ languages including all major Indic languages.
    """

    def __init__(
        self,
        language: str = "hi",
        device: str = "cuda",
    ):
        super().__init__(language=language)
        self.device = device
        self._model = None
        self._processor = None
        self._tokenizer = None

    def load(self) -> None:
        """Load MMS alignment model."""
        if self._loaded:
            return

        try:
            import torch
            from transformers import Wav2Vec2ForCTC, AutoProcessor
        except ImportError:
            raise ImportError(
                "transformers and torch are required. Install with: "
                "pip install transformers torch"
            )

        # Load MMS-FA model
        model_id = "facebook/mms-1b-all"

        self._processor = AutoProcessor.from_pretrained(model_id)
        self._model = Wav2Vec2ForCTC.from_pretrained(model_id)

        if self.device == "cuda":
            import torch
            if torch.cuda.is_available():
                self._model = self._model.cuda()

        self._model.eval()
        self._loaded = True

    def _get_mms_lang_code(self, language: str) -> str:
        """Convert language code to MMS format."""
        return MMS_LANGUAGE_MAP.get(language, language)

    def align(
        self,
        audio: np.ndarray,
        sample_rate: int,
        transcript: str,
        language: Optional[str] = None,
    ) -> AlignmentResult:
        """
        Align transcript to audio using MMS.

        Uses CTC forced alignment to get word-level timestamps and scores.
        """
        if not self._loaded:
            self.load()

        import torch

        lang = language or self.language
        mms_lang = self._get_mms_lang_code(lang)

        # Set language for processor
        try:
            self._processor.tokenizer.set_target_lang(mms_lang)
            self._model.load_adapter(mms_lang)
        except Exception as e:
            # Fall back to closest available language
            pass

        # Ensure correct sample rate (MMS expects 16kHz)
        if sample_rate != 16000:
            from ..utils.audio import resample_audio
            audio = resample_audio(audio, sample_rate, 16000)
            sample_rate = 16000

        # Ensure float32
        if audio.dtype != np.float32:
            audio = audio.astype(np.float32)

        # Process audio
        inputs = self._processor(
            audio,
            sampling_rate=sample_rate,
            return_tensors="pt",
            padding=True,
        )

        if self.device == "cuda" and torch.cuda.is_available():
            inputs = {k: v.cuda() for k, v in inputs.items()}

        # Get CTC emissions
        with torch.no_grad():
            outputs = self._model(**inputs)
            emissions = outputs.logits

        # Get emission probabilities
        emissions = torch.log_softmax(emissions, dim=-1)

        # Tokenize transcript
        tokens = self._processor.tokenizer(transcript, return_tensors="pt")
        token_ids = tokens.input_ids[0].tolist()

        # Perform CTC forced alignment
        aligned_words = self._ctc_forced_align(
            emissions[0].cpu().numpy(),
            transcript,
            token_ids,
            audio_duration=len(audio) / sample_rate,
        )

        # Calculate metrics
        if aligned_words:
            scores = [w.score for w in aligned_words]
            overall_score = np.mean(scores)
            avg_score = np.mean(scores)
            min_score = np.min(scores)
            coverage = len([w for w in aligned_words if w.score > 0.1]) / max(1, len(aligned_words))
        else:
            overall_score = 0.0
            avg_score = 0.0
            min_score = 0.0
            coverage = 0.0

        timing_issues = self.validate_timing(aligned_words)

        # Find unaligned words (very low score)
        unaligned = [w.word for w in aligned_words if w.score < 0.1]

        return AlignmentResult(
            words=aligned_words,
            overall_score=overall_score,
            coverage=coverage,
            duration=len(audio) / sample_rate,
            avg_word_score=avg_score,
            min_word_score=min_score,
            unaligned_words=unaligned,
            timing_issues=timing_issues,
        )

    def _ctc_forced_align(
        self,
        emissions: np.ndarray,
        transcript: str,
        token_ids: List[int],
        audio_duration: float,
    ) -> List[AlignedWord]:
        """
        Perform CTC forced alignment.

        This is a simplified version - for production, use torchaudio's
        forced alignment or MFA for more accurate results.
        """
        # Split transcript into words
        words = transcript.split()
        if not words:
            return []

        n_frames = emissions.shape[0]
        frame_duration = audio_duration / n_frames

        # Simple alignment: distribute frames proportionally by word length
        total_chars = sum(len(w) for w in words)
        if total_chars == 0:
            return []

        aligned = []
        current_frame = 0

        for word in words:
            # Allocate frames proportionally
            word_frames = int((len(word) / total_chars) * n_frames)
            word_frames = max(1, word_frames)

            start_frame = current_frame
            end_frame = min(current_frame + word_frames, n_frames)

            # Get average emission score for this segment
            if start_frame < end_frame:
                segment_emissions = emissions[start_frame:end_frame]
                # Use max probability at each frame as score proxy
                max_probs = np.max(segment_emissions, axis=-1)
                score = float(np.mean(np.exp(max_probs)))
            else:
                score = 0.0

            aligned.append(AlignedWord(
                word=word,
                start=start_frame * frame_duration,
                end=end_frame * frame_duration,
                score=score,
            ))

            current_frame = end_frame

        return aligned

class TorchAudioAligner(ForcedAligner):
    """
    Forced aligner using torchaudio's CTC forced alignment.

    More accurate than the simple MMS aligner above.
    """

    def __init__(self, language: str = "hi", device: str = "cuda"):
        super().__init__(language=language)
        self.device = device
        self._bundle = None

    def load(self) -> None:
        """Load torchaudio alignment model."""
        if self._loaded:
            return

        try:
            import torch
            import torchaudio
            from torchaudio.pipelines import MMS_FA as bundle
        except ImportError:
            raise ImportError(
                "torchaudio is required. Install with: pip install torchaudio"
            )

        self._bundle = bundle
        self._model = bundle.get_model()


**Identifying critical flaw in alignment scoring**

The alignment algorithm ignores transcript content and bases scores solely on audio emissions, making alignment scores meaningless for distinguishing transcripts; this undermines consensus methods relying on alignment, like WeightedROVER. Next, I'll analyze Gemini's high WER including tokenization mismatches and investigate Gemini model usage in its code.


**Inspecting Gemini model code for prompt effects**


sed -n '1,260p' transcribe/models/gemini.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""Gemini model wrapper for transcription."""

import time
import json
import base64
import tempfile
from typing import Optional, Dict, Any
import numpy as np

from .base import BaseTranscriber, TranscriptionResult, WordSegment
from ..prompts import SYSTEM_PROMPT, OUTPUT_SCHEMA, LANGUAGE_CODES

class GeminiTranscriber(BaseTranscriber):
    """Gemini transcription using Google's API."""

    def __init__(
        self,
        api_key: str,
        model_name: str = "gemini-2.0-flash",
        temperature: float = 0.0,
    ):
        super().__init__(model_name=f"gemini-{model_name.split('-')[-1]}")
        self.api_key = api_key
        self.model_id = model_name
        self.temperature = temperature
        self._client = None

    def load(self) -> None:
        """Initialize Gemini client."""
        if self._loaded:
            return

        try:
            import google.generativeai as genai
        except ImportError:
            raise ImportError(
                "google-generativeai is required. Install with: "
                "pip install google-generativeai"
            )

        genai.configure(api_key=self.api_key)
        self._client = genai
        # Use simple text mode for reliability
        self._model = genai.GenerativeModel(
            self.model_id,
            generation_config=genai.GenerationConfig(
                temperature=self.temperature,
            ),
        )
        self._loaded = True

    def _prepare_audio(self, audio: np.ndarray, sample_rate: int) -> bytes:
        """Convert audio array to bytes for upload."""
        import soundfile as sf
        import io

        # Ensure float32
        if audio.dtype != np.float32:
            audio = audio.astype(np.float32)

        # Write to WAV bytes
        buffer = io.BytesIO()
        sf.write(buffer, audio, sample_rate, format="WAV")
        buffer.seek(0)
        return buffer.read()

    def transcribe(
        self,
        audio: np.ndarray,
        sample_rate: int,
        language: Optional[str] = None,
        **kwargs,
    ) -> TranscriptionResult:
        """
        Transcribe audio using Gemini.

        Args:
            audio: Audio array (mono, float32)
            sample_rate: Sample rate
            language: Optional language hint
            **kwargs: Additional arguments

        Returns:
            TranscriptionResult with both mixed and native transcripts
        """
        if not self._loaded:
            self.load()

        start_time = time.time()

        # Prepare audio
        audio_bytes = self._prepare_audio(audio, sample_rate)

        # Create language-specific prompt if needed
        lang_hint = language or "auto-detect"
        if language and language in LANGUAGE_CODES:
            lang_hint = list(LANGUAGE_CODES.keys())[
                list(LANGUAGE_CODES.values()).index(language)
            ] if language in LANGUAGE_CODES.values() else language

        # Build simple prompt for reliable transcription
        script_map = {
            "telugu": "Telugu script (తెలుగు)",
            "hindi": "Devanagari script (हिंदी)",
            "tamil": "Tamil script (தமிழ்)",
            "kannada": "Kannada script (ಕನ್ನಡ)",
            "malayalam": "Malayalam script (മലയാളം)",
            "bengali": "Bengali script (বাংলা)",
            "gujarati": "Gujarati script (ગુજરાતી)",
            "marathi": "Devanagari script (मराठी)",
            "punjabi": "Gurmukhi script (ਪੰਜਾਬੀ)",
            "odia": "Odia script (ଓଡ଼ିଆ)",
            "assamese": "Assamese script (অসমীয়া)",
            "english": "Latin script",
        }
        script = script_map.get(lang_hint.lower(), "native script")

        prompt = f"""Transcribe this {lang_hint} audio EXACTLY as spoken.

RULES:
1. Use {script} for {lang_hint} words
2. Keep English words in Latin script if code-mixed
3. Do NOT add or remove any words
4. Do NOT correct grammar
5. Transcribe verbatim including disfluencies

Return ONLY the transcription text, nothing else."""

        # Upload audio and transcribe
        try:
            # Upload file
            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
                f.write(audio_bytes)
                temp_path = f.name

            try:
                audio_file = self._client.upload_file(temp_path)

                # Generate transcription with simple prompt
                response = self._model.generate_content([audio_file, prompt])

                # Get plain text response
                text = response.text.strip()

            finally:
                import os
                os.unlink(temp_path)
                # Clean up uploaded file
                try:
                    audio_file.delete()
                except:
                    pass

        except Exception as e:
            # Return empty result on error
            import logging
            logging.warning(f"Gemini transcription failed: {e}")
            return TranscriptionResult(
                text="",
                language=language or "unknown",
                confidence=0.0,
                model_name=self.model_name,
                processing_time=time.time() - start_time,
                audio_duration=len(audio) / sample_rate,
                raw_output={"error": str(e)},
            )

        processing_time = time.time() - start_time
        audio_duration = len(audio) / sample_rate

        return TranscriptionResult(
            text=text,
            text_native=text,  # Same for now with simple mode
            language=language or "unknown",
            confidence=0.9,
            words=[],
            segments=[{
                "start": 0.0,
                "end": audio_duration,
                "text": text,
            }],
            model_name=self.model_name,
            processing_time=processing_time,
            audio_duration=audio_duration,
            contains_code_mixing=False,
            languages_present=[language] if language else [],
            raw_output={"text": text},
        )

class GeminiBatchTranscriber(GeminiTranscriber):
    """
    Batch transcription with Gemini for cost efficiency.
    Collects requests and processes them in batches.
    """

    def __init__(
        self,
        api_key: str,
        model_name: str = "gemini-2.0-flash",
        temperature: float = 0.0,
        batch_size: int = 10,
    ):
        super().__init__(api_key, model_name, temperature)
        self.batch_size = batch_size
        self._pending = []
        self._results = {}

    def add_to_batch(
        self,
        audio: np.ndarray,
        sample_rate: int,
        audio_id: str,
        language: Optional[str] = None,
    ) -> None:
        """Add audio to pending batch."""
        self._pending.append({
            "audio": audio,
            "sample_rate": sample_rate,
            "audio_id": audio_id,
            "language": language,
        })

    def process_batch(self) -> Dict[str, TranscriptionResult]:
        """Process all pending items."""
        results = {}
        for item in self._pending:
            result = self.transcribe(
                item["audio"],
                item["sample_rate"],
                item["language"],
            )
            results[item["audio_id"]] = result

        self._pending = []
        return results


**Noticing number normalization missing in WER**

The existing normalization removes punctuation and lowercases but doesn't actually normalize numbers despite the comment suggesting it should. This likely contributes to inflated WER, especially since references and ASR outputs handle numbers inconsistently, sometimes digits, sometimes spelled out in native scripts. Addressing number normalization could reduce WER errors caused by formatting mismatches.


**Identifying tokenization and alignment issues inflating Gemini WER**

Gemini’s higher WER largely stems from tokenization and morphological segmentation differences in Telugu, causing space-based WER to over-penalize minor spacing variations. This suggests using character error rate or script-aware tokenization instead. The consensus (ROVER) pipeline’s simplified alignment and lack of normalization also lead to misalignments and poor consensus choice. Analyzing worst WER cases with normalization tweaks and comparing transcripts will clarify fix paths, including improved normalization, morphological merging, and alignment refinements.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""Configuration management for the transcription pipeline."""

import os
from pathlib import Path
from dataclasses import dataclass, field
from typing import Optional
from dotenv import load_dotenv

# Load environment variables
load_dotenv(Path(__file__).parent.parent / ".env")

@dataclass
class R2Config:
    """Cloudflare R2 configuration."""
    endpoint_url: str = os.getenv("R2_ENDPOINT_URL", "")
    access_key_id: str = os.getenv("R2_ACCESS_KEY_ID", "")
    secret_access_key: str = os.getenv("R2_SECRET_ACCESS_KEY", "")
    account_id: str = os.getenv("ACCOUNT_ID", "")

    # Buckets
    eval_bucket: str = "xcodec"
    eval_prefix: str = "evaluation/"

@dataclass
class ModelConfig:
    """Model configuration."""
    # Whisper
    whisper_model: str = "large-v3"
    whisper_device: str = "cuda"
    whisper_compute_type: str = "float16"

    # IndicConformer
    indicconformer_model: str = "ai4bharat/indicconformer_stt_hi_hybrid_rnnt_large"

    # Gemini
    gemini_model: str = "gemini-2.0-flash"
    gemini_api_key: str = os.getenv("GEMINI", "")
    gemini_temperature: float = 0.0

    # MMS Aligner
    mms_aligner_model: str = "facebook/mms-fa"

@dataclass
class PipelineConfig:
    """Pipeline configuration."""
    # Agreement thresholds
    high_confidence_threshold: float = 0.85
    alignment_score_threshold: float = 0.7

    # ROVER weights by language (indicconformer, whisper, gemini)
    language_weights: dict = field(default_factory=lambda: {
        "hindi": (0.45, 0.35, 0.20),
        "telugu": (0.50, 0.30, 0.20),
        "tamil": (0.50, 0.30, 0.20),
        "kannada": (0.50, 0.30, 0.20),
        "malayalam": (0.50, 0.30, 0.20),
        "bengali": (0.45, 0.35, 0.20),
        "gujarati": (0.45, 0.35, 0.20),
        "marathi": (0.45, 0.35, 0.20),
        "punjabi": (0.45, 0.35, 0.20),
        "odia": (0.50, 0.30, 0.20),
        "assamese": (0.50, 0.30, 0.20),
        "english": (0.20, 0.50, 0.30),
        "code_mixed": (0.30, 0.30, 0.40),
    })

    # Processing
    chunk_duration_s: float = 10.0
    chunk_overlap_s: float = 2.0
    max_segment_duration_s: float = 30.0

    # Repair loop
    max_repair_iterations: int = 3
    repair_threshold: float = 0.6

@dataclass
class Config:
    """Main configuration container."""
    r2: R2Config = field(default_factory=R2Config)
    model: ModelConfig = field(default_factory=ModelConfig)
    pipeline: PipelineConfig = field(default_factory=PipelineConfig)

    # Paths
    base_dir: Path = Path(__file__).parent.parent
    data_dir: Path = field(default_factory=lambda: Path(__file__).parent.parent / "transcribe" / "data")
    eval_dir: Path = field(default_factory=lambda: Path(__file__).parent.parent / "transcribe" / "data" / "eval")
    cache_dir: Path = field(default_factory=lambda: Path(__file__).parent.parent / "transcribe" / "cache")

# Global config instance
config = Config()


**Preparing WER analysis script**

I'm planning to create a Python script that reads the data file, computes WER differences between Gemini and Indic/Consensus for two languages with 10 samples each, and highlights the worst cases along with tokenization and alignment details for further analysis.


python3 - <<'PY'
import json
from pathlib import Path
from transcribe.utils.text_processing import normalize_for_wer, compute_wer

path = Path('indicwhisper_eval_10samples.json')
obj = json.loads(path.read_text())

rows = []
for lang, samples in obj.items():
    for s in samples:
        rows.append((lang, s['sample_id'], s['reference'], s['indicwhisper_hyp'], s['gemini_hyp'], s['consensus_hyp'], s['indicwhisper_wer'], s['gemini_wer'], s['consensus_wer']))

# sort by gemini_wer desc
rows_sorted = sorted(rows, key=lambda r: r[7], reverse=True)
print('Top Gemini WER cases:')
for lang, sid, ref, iw, gem, cons, iw_wer, gem_wer, cons_wer in rows_sorted[:6]:
    print('\n', lang, sid, 'gem_wer', gem_wer, 'iw_wer', iw_wer, 'cons', cons_wer)
    print('REF:', ref)
    print('GEM:', gem)
    print('IW :', iw)

# sort by consensus_wer desc
rows_sorted = sorted(rows, key=lambda r: r[8], reverse=True)
print('\nTop Consensus WER cases:')
for lang, sid, ref, iw, gem, cons, iw_wer, gem_wer, cons_wer in rows_sorted[:6]:
    print('\n', lang, sid, 'cons_wer', cons_wer, 'iw_wer', iw_wer, 'gem_wer', gem_wer)
    print('REF:', ref)
    print('CONS:', cons)
    print('IW  :', iw)
    print('GEM :', gem)

PY


Output:
Exit code: 1
Wall time: 0.2 seconds
Output:
Traceback (most recent call last):
  File "<stdin>", line 11, in <module>
TypeError: string indices must be integers


**Investigating JSON file format**

I'm checking whether the file is pure JSON, JSON Lines, or malformed since errors suggest the loaded object might be a list or string, so I'll inspect its start and end to understand its structure.


python3 - <<'PY'
from pathlib import Path
import json
text = Path('indicwhisper_eval_10samples.json').read_text()
print(text[:200])
print('---')
print(text[-200:])
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
{
  "telugu": [
    {
      "sample_id": "telugu_000000",
      "language": "telugu",
      "reference": "చిన్న ద్వీపాలలో చాలా వరకు స్వతంత్ర దేశాలు లేదా ఫ్రాన్స్ తో సంబంధం కలిగి ఉన్నాయి ఇంకా వీటిని లగ
---
080624,
    "consensus_mean_wer": 0.15921417061403187,
    "indicwhisper_mean_cer": 0.0804860767921819,
    "gemini_mean_cer": 0.033965569727092984,
    "consensus_mean_cer": 0.07892934744153875
  }
}


**Identifying JSON structure issue**

The JSON top-level has keys whose values vary: some are lists (samples) and some are dicts (like summary), causing a type error when iterating all values as lists. Adjusting the script to process only list-type values will fix this.


python3 - <<'PY'
import json
from pathlib import Path

obj = json.loads(Path('indicwhisper_eval_10samples.json').read_text())

rows = []
for lang, samples in obj.items():
    if not isinstance(samples, list):
        continue
    for s in samples:
        rows.append((lang, s['sample_id'], s['reference'], s['indicwhisper_hyp'], s['gemini_hyp'], s['consensus_hyp'], s['indicwhisper_wer'], s['gemini_wer'], s['consensus_wer']))

rows_sorted = sorted(rows, key=lambda r: r[7], reverse=True)
print('Top Gemini WER cases:')
for lang, sid, ref, iw, gem, cons, iw_wer, gem_wer, cons_wer in rows_sorted[:6]:
    print('\n', lang, sid, 'gem_wer', gem_wer, 'iw_wer', iw_wer, 'cons', cons_wer)
    print('REF:', ref)
    print('GEM:', gem)
    print('IW :', iw)

rows_sorted = sorted(rows, key=lambda r: r[8], reverse=True)
print('\nTop Consensus WER cases:')
for lang, sid, ref, iw, gem, cons, iw_wer, gem_wer, cons_wer in rows_sorted[:6]:
    print('\n', lang, sid, 'cons_wer', cons_wer, 'iw_wer', iw_wer, 'gem_wer', gem_wer)
    print('REF:', ref)
    print('CONS:', cons)
    print('IW  :', iw)
    print('GEM :', gem)

PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
Top Gemini WER cases:

 telugu telugu_000007 gem_wer 0.5 iw_wer 0.1 cons 0.5
REF: బుధవారం జరిగిన ఈవెంట్కు ముందు carpanedo ఛాంపియన్షిప్లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు
GEM: బుధవారం జరిగిన ఈవెంట్ కు ముందు కార్పనేడో ఛాంపియన్షిప్ లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు.
IW : బుధవారం జరిగిన ఈవెంట్కు ముందు కార్పనేడో ఛాంపియన్షిప్లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు

 telugu telugu_000004 gem_wer 0.35714285714285715 iw_wer 0.0 cons 0.0
REF: పరికరాన్ని గాల్లో కదిలించడం ద్వారా వీడియో గేమ్స్లో యాక్షన్లు మరియు కదలికలను నియంత్రించడానికి ఇది ఆటగాళ్లకు వీలు కల్పిస్తుంది
GEM: పరికరాని గాలిలో కదిలించడం ద్వారా వీడియో గేమ్స్ లో యాక్షన్లు మరియు కదలికలను నియంత్రించడానికి ఇది ఆటగాళ్ళకు వీలు కల్పిస్తుంది.
IW : పరికరాన్ని గాల్లో కదిలించడం ద్వారా వీడియో గేమ్స్లో యాక్షన్లు మరియు కదలికలను నియంత్రించడానికి ఇది ఆటగాళ్లకు వీలు కల్పిస్తుంది

 hindi hindi_000004 gem_wer 0.2777777777777778 iw_wer 0.05555555555555555 cons 0.2777777777777778
REF: इंटरनेट पर यह खोज शत्रुतापूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी
GEM: इंटरनेट पर यह खोज शत्रुता पूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी। सही है।
IW : इंटरनेट पर ये खोज शत्रुतापूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी

 telugu telugu_000008 gem_wer 0.2631578947368421 iw_wer 0.15789473684210525 cons 0.2631578947368421
REF: పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడ్డాయి అయితే పైభాగంలో ఇనుప ఖనిజం ఉంటే ట్రాక్ లు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు
GEM: పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడాయి. అయితే పై భాగంలో ఇనుప ఖనిజం ఉంటే ట్రాకులు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు.
IW : పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడాయి అయితే పైభాగంలో ఇనుప ఖనిజం ఉంటే ట్రాక్లు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు

 hindi hindi_000009 gem_wer 0.21428571428571427 iw_wer 0.10714285714285714 cons 0.10714285714285714
REF: पुरुषों के स्टैंडिंग सुपर-जी में ऑस्ट्रेलिया के मिशेल गौरले ग्यारहवें स्थान पर रहे चेक प्रतियोगी ओल्डरिच जेलिनेक पुरुषों के सिटिंग सुपर-जी में सोलहवें स्थान पर रहे
GEM: पुरुषों के स्टैंडिंग सुपर जी में ऑस्ट्रेलिया के मिशेल गोरले 11वें स्थान पर रहे। चेक प्रतियोगी ऑल्ड्रिच जेलिनेक पुरुषों के सिटिंग सुपर जी में 16वें स्थान पर रहे।
IW : पुरुषों के स्टैंडिंग सुपर जी में ऑस्ट्रेलिया के मिशेल गॉर्ले ग्यारहवें स्थान पर रहे चेक प्रतियोगी ऑल्ड्रिज जेलिनक पुरुषों के सिटिंग सुपर जी में सोलहवें स्थान पर रहे

 telugu telugu_000001 gem_wer 0.2 iw_wer 0.1 cons 0.1
REF: కొన్ని క్రియలు ఆబ్జెక్టుల మధ్య తేడాను గుర్తించడానికి ఇది ఒక ముఖ్యమైన మార్గం
GEM: కొన్ని క్రియలు ఆబ్జెక్టులు మధ్య తేడాలను గుర్తించడానికి ఇది ఒక ముఖ్యమైన మార్గం.
IW : కొన్ని క్రియలు అబ్జెక్టుల మధ్య తేడాను గుర్తించడానికి ఇది ఒక ముఖ్యమైన మార్గం

Top Consensus WER cases:

 telugu telugu_000009 cons_wer 1.1764705882352942 iw_wer 1.1764705882352942 gem_wer 0.11764705882352941
REF: ఇది బేస్ స్టేషన్ లో డ్యూయల్ రేడియోలు ఉంటే 802.11a 802.11b మరియు 802.11gతో అనుకూలంగా వెనుకకు అనుమతిస్తుంది
CONS: ఇది బేస్ స్టేషన్లో డ్యూయల్ రేడియోలు ఉంటే ఎనిమిది వందల రెండు పాయింట్ పదకొండు ఏ ఎనిమిది వందల రెండు పాయింట్ పదకొండు బి మరియు ఎనిమిది వందల రెండు పాయింట్ పదకొండు జితో అనుకూలంగా వెనుకకు అనుమతిస్తుంది
IW  : ఇది బేస్ స్టేషన్లో డ్యూయల్ రేడియోలు ఉంటే ఎనిమిది వందల రెండు పాయింట్ పదకొండు ఏ ఎనిమిది వందల రెండు పాయింట్ పదకొండు బి మరియు ఎనిమిది వందల రెండు పాయింట్ పదకొండు జితో అనుకూలంగా వెనుకకు అనుమతిస్తుంది
GEM : ఇది బేస్ స్టేషన్ లో డ్యూయల్ రేడియోలు ఉంటే 802.11a, 802.11b మరియు 802.11g తో అనుకూలంగా వెనుకకు అనుమతిస్తుంది.

 telugu telugu_000007 cons_wer 0.5 iw_wer 0.1 gem_wer 0.5
REF: బుధవారం జరిగిన ఈవెంట్కు ముందు carpanedo ఛాంపియన్షిప్లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు
CONS: బుధవారం జరిగిన ఈవెంట్ కు ముందు కార్పనేడో ఛాంపియన్షిప్ లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు.
IW  : బుధవారం జరిగిన ఈవెంట్కు ముందు కార్పనేడో ఛాంపియన్షిప్లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు
GEM : బుధవారం జరిగిన ఈవెంట్ కు ముందు కార్పనేడో ఛాంపియన్షిప్ లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు.

 hindi hindi_000004 cons_wer 0.2777777777777778 iw_wer 0.05555555555555555 gem_wer 0.2777777777777778
REF: इंटरनेट पर यह खोज शत्रुतापूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी
CONS: इंटरनेट पर यह खोज शत्रुता पूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी। सही है।
IW  : इंटरनेट पर ये खोज शत्रुतापूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी
GEM : इंटरनेट पर यह खोज शत्रुता पूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी। सही है।

 telugu telugu_000008 cons_wer 0.2631578947368421 iw_wer 0.15789473684210525 gem_wer 0.2631578947368421
REF: పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడ్డాయి అయితే పైభాగంలో ఇనుప ఖనిజం ఉంటే ట్రాక్ లు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు
CONS: పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడాయి. అయితే పై భాగంలో ఇనుప ఖనిజం ఉంటే ట్రాకులు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు.
IW  : పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడాయి అయితే పైభాగంలో ఇనుప ఖనిజం ఉంటే ట్రాక్లు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు
GEM : పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడాయి. అయితే పై భాగంలో ఇనుప ఖనిజం ఉంటే ట్రాకులు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు.

 hindi hindi_000006 cons_wer 0.2 iw_wer 0.1 gem_wer 0.2
REF: इसे केमिकल का ph कहा जाता है आप लाल गोभी के जूस का इस्तेमाल करके एक संकेतक बना सकते हैं
CONS: इसे केमिकल का पीएच कहा जाता है। आप लाल गोभी के जूस को इस्तेमाल करके एक संकेतक बना सकते हैं।
IW  : इसे केमिकल का पीएच कहा जाता है आप लाल गोभी के जूस को इस्तेमाल करके एक संकेतक बना सकते हैं
GEM : इसे केमिकल का पीएच कहा जाता है। आप लाल गोभी के जूस को इस्तेमाल करके एक संकेतक बना सकते हैं।

 hindi hindi_000008 cons_wer 0.13157894736842105 iw_wer 0.10526315789473684 gem_wer 0.13157894736842105
REF: यह संबंधित है लेकिन आम तौर पर इसमें अल्पाइन शैली की स्की टूरिंग या माउंटेनीयरिंग शामिल नहीं होती है जिसे खड़े इलाकों में किया जाता है और जिसके लिए बहुत कड़ी स्की और बूट की ज़रूरत होती है
CONS: यह संबंधित है लेकिन आमतौर पर इसमें अल्पाइन शैली की स्की टूरिंग या माउंटेनियरिंग शामिल नहीं होती है जिसे खड़े इलाकों में किया जाता है और जिसके लिए बहुत कड़ी स्की और बूट की जरूरत होती है।
IW  : यह संबंधित है लेकिन आमतौर पर इसमें अल्पाइन शैली की स्की टूरिंग या माउंटेनियरिंग शामिल नहीं होती है जिसे खड़े इलाकों में किया जाता है और जिसके लिए बहुत कड़ी स्की और बूट की जरूरत होती है
GEM : यह संबंधित है लेकिन आमतौर पर इसमें अल्पाइन शैली की स्की टूरिंग या माउंटेनियरिंग शामिल नहीं होती है जिसे खड़े इलाकों में किया जाता है और जिसके लिए बहुत कड़ी स्की और बूट की जरूरत होती है।


**Identifying WER inflation causes**

WER is artificially high due to script mismatches, tokenization differences, digit vs. spelled number inconsistencies, and occasional extra hallucinated sentences in Gemini outputs, suggesting a need to normalize scripts and number formats before evaluation.


**Investigating prompt and evaluation fixes for script mismatches**

Gemini may hallucinate filler like "सही है." possibly from prompt behavior, and script mismatches (Latin vs. Telugu) cause unfair WER inflation; adjusting prompts seems tough, so improving evaluation by transliterating or normalizing tokens before WER calculation is promising to ensure fairer scoring.


sed -n '1,220p' transcribe/evaluation/metrics.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""Evaluation metrics for transcription."""

from typing import List, Dict, Any, Optional
from dataclasses import dataclass
import numpy as np

from ..utils.text_processing import compute_wer, compute_cer, normalize_text

@dataclass
class MetricsResult:
    """Metrics for a single sample."""
    wer: float
    cer: float
    substitutions: int
    insertions: int
    deletions: int
    ref_words: int
    hyp_words: int
    ref_chars: int
    hyp_chars: int

    def to_dict(self) -> dict:
        return {
            "wer": self.wer,
            "cer": self.cer,
            "substitutions": self.substitutions,
            "insertions": self.insertions,
            "deletions": self.deletions,
            "ref_words": self.ref_words,
            "hyp_words": self.hyp_words,
            "ref_chars": self.ref_chars,
            "hyp_chars": self.hyp_chars,
        }

def compute_metrics(
    reference: str,
    hypothesis: str,
    normalize: bool = True,
) -> MetricsResult:
    """
    Compute WER and CER metrics.

    Args:
        reference: Ground truth text
        hypothesis: Predicted text
        normalize: Whether to normalize texts before comparison

    Returns:
        MetricsResult with WER, CER, and detailed counts
    """
    if normalize:
        reference = normalize_text(reference)
        hypothesis = normalize_text(hypothesis)

    wer_result = compute_wer(reference, hypothesis)
    cer_result = compute_cer(reference, hypothesis)

    return MetricsResult(
        wer=wer_result["wer"],
        cer=cer_result["cer"],
        substitutions=wer_result["substitutions"],
        insertions=wer_result["insertions"],
        deletions=wer_result["deletions"],
        ref_words=wer_result["ref_words"],
        hyp_words=wer_result["hyp_words"],
        ref_chars=cer_result["ref_chars"],
        hyp_chars=cer_result["hyp_chars"],
    )

@dataclass
class AggregatedMetrics:
    """Aggregated metrics across multiple samples."""
    # WER metrics
    mean_wer: float
    median_wer: float
    std_wer: float
    min_wer: float
    max_wer: float

    # CER metrics
    mean_cer: float
    median_cer: float
    std_cer: float

    # Micro-averaged WER (total errors / total words)
    micro_wer: float
    micro_cer: float

    # Counts
    total_samples: int
    total_ref_words: int
    total_hyp_words: int
    total_errors: int

    # Distribution
    wer_percentiles: Dict[int, float]  # 5, 25, 50, 75, 95

    def to_dict(self) -> dict:
        return {
            "mean_wer": self.mean_wer,
            "median_wer": self.median_wer,
            "std_wer": self.std_wer,
            "min_wer": self.min_wer,
            "max_wer": self.max_wer,
            "mean_cer": self.mean_cer,
            "median_cer": self.median_cer,
            "std_cer": self.std_cer,
            "micro_wer": self.micro_wer,
            "micro_cer": self.micro_cer,
            "total_samples": self.total_samples,
            "total_ref_words": self.total_ref_words,
            "total_hyp_words": self.total_hyp_words,
            "total_errors": self.total_errors,
            "wer_percentiles": self.wer_percentiles,
        }

def aggregate_metrics(
    metrics_list: List[MetricsResult],
) -> AggregatedMetrics:
    """
    Aggregate metrics across multiple samples.

    Args:
        metrics_list: List of MetricsResult objects

    Returns:
        AggregatedMetrics with summary statistics
    """
    if not metrics_list:
        return AggregatedMetrics(
            mean_wer=0.0, median_wer=0.0, std_wer=0.0, min_wer=0.0, max_wer=0.0,
            mean_cer=0.0, median_cer=0.0, std_cer=0.0,
            micro_wer=0.0, micro_cer=0.0,
            total_samples=0, total_ref_words=0, total_hyp_words=0, total_errors=0,
            wer_percentiles={},
        )

    wers = [m.wer for m in metrics_list]
    cers = [m.cer for m in metrics_list]

    # Totals for micro-averaging
    total_ref_words = sum(m.ref_words for m in metrics_list)
    total_hyp_words = sum(m.hyp_words for m in metrics_list)
    total_ref_chars = sum(m.ref_chars for m in metrics_list)
    total_hyp_chars = sum(m.hyp_chars for m in metrics_list)
    total_word_errors = sum(m.substitutions + m.insertions + m.deletions for m in metrics_list)

    # Micro-averaged metrics
    micro_wer = total_word_errors / total_ref_words if total_ref_words > 0 else 0.0

    # Estimate character errors for micro CER
    total_char_errors = sum(
        int(m.cer * m.ref_chars) for m in metrics_list
    )
    micro_cer = total_char_errors / total_ref_chars if total_ref_chars > 0 else 0.0

    # Percentiles
    wer_array = np.array(wers)
    percentiles = {
        5: float(np.percentile(wer_array, 5)),
        25: float(np.percentile(wer_array, 25)),
        50: float(np.percentile(wer_array, 50)),
        75: float(np.percentile(wer_array, 75)),
        95: float(np.percentile(wer_array, 95)),
    }

    return AggregatedMetrics(
        mean_wer=float(np.mean(wers)),
        median_wer=float(np.median(wers)),
        std_wer=float(np.std(wers)),
        min_wer=float(np.min(wers)),
        max_wer=float(np.max(wers)),
        mean_cer=float(np.mean(cers)),
        median_cer=float(np.median(cers)),
        std_cer=float(np.std(cers)),
        micro_wer=micro_wer,
        micro_cer=micro_cer,
        total_samples=len(metrics_list),
        total_ref_words=total_ref_words,
        total_hyp_words=total_hyp_words,
        total_errors=total_word_errors,
        wer_percentiles=percentiles,
    )

def compute_per_language_metrics(
    results: List[Dict[str, Any]],
) -> Dict[str, AggregatedMetrics]:
    """
    Compute metrics grouped by language.

    Args:
        results: List of dicts with 'language', 'reference', 'hypothesis' keys

    Returns:
        Dict mapping language -> AggregatedMetrics
    """
    by_language: Dict[str, List[MetricsResult]] = {}

    for r in results:
        lang = r.get("language", "unknown")
        ref = r.get("reference", "")
        hyp = r.get("hypothesis", "")

        metrics = compute_metrics(ref, hyp)

        if lang not in by_language:
            by_language[lang] = []
        by_language[lang].append(metrics)

    return {
        lang: aggregate_metrics(metrics_list)
        for lang, metrics_list in by_language.items()
    }


Summary: auto


**Noticing evaluation inconsistency and next steps**

Evaluation uses different text normalization methods across metrics, potentially causing inconsistent WER results; next steps include investigating why Gemini's WER is higher, analyzing consensus and ROVER pipelines, comparing error cases, and reviewing relevant docs to align with the outlined plan.


**Planning quick doc review for consensus details**

I'll briefly check the top sections of GPT and Gemini docs to spot any evaluation or consensus guidance relevant to aligning the pipeline.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
## What forced alignment is (and isn’t) in this setup

Forced alignment is a **validator + locator**: given *(audio, transcript)* it finds the best timing path and can output **word/phoneme timestamps** plus **alignment scores**. It is extremely good at telling you “this text fits the audio well” vs “this text is off,” but it won’t magically correct a bad transcript unless you use it inside a **repair loop**.

Tools that fit your language set:

* **Montreal Forced Aligner (MFA)**: Kaldi-based, trainable, supports G2P via Pynini; best when you can provide/learn lexicons per language. ([montreal-forced-aligner.readthedocs.io][1])
* **WhisperX**: runs Whisper, then refines timestamps using **wav2vec2 forced alignment** (word-level timestamps + “unaligned word” signals). ([GitHub][2])
* **MMS CTC forced aligner (158 languages)**: practical for Indic because it supports **ISO-639-3 language codes** and can **romanize** during preprocessing (useful across scripts). ([huggingface.co][3])
* **torchaudio CTC forced alignment API**: good if you want to build your own scoring/diagnostics loop (CTC segmentation + high-level bundles). ([docs.pytorch.org][4])

## The “tight” strategy: Alignment-weighted system combination + targeted repair

You said you have **3 transcripts per utterance** (Whisper-large-v3, Indic Conformer(s), LLM-ASR, etc.). The highest-yield approach is:

### 0) Normalize aggressively (or your voting is fake)

Do this *before* any comparison/alignment:

* Script + punctuation normalization (Indic punctuation, danda, quotes)
* Numerals (e.g., “10”, “పది”, “दस”) → choose one canonical form per language
* Common expansions (abbrev, currency, dates) in a consistent style
* Remove obvious ASR junk tokens (“[MUSIC]”, “uh”, repeated words) only if you plan to remove them everywhere

If normalization is inconsistent, ROVER/consensus will “disagree” for dumb reasons.

---

### 1) Score each candidate transcript with an **independent forced aligner**

For each candidate transcript (T_j) for the same audio:

1. Run a forced aligner (recommend **MMS CTC aligner** as the universal baseline, and optionally MFA for languages where you can build strong lexicons). ([huggingface.co][3])
2. Extract these metrics (you’ll use them as weights + filters):

* **Coverage**: % of words/chars that receive a valid alignment (unaligned words are a huge red flag; WhisperX exposes this concept too). ([huggingface.co][5])
* **Mean alignment score** (CTC path scores / emission-based score; MMS/torchaudio-style aligners give you per-token/segment scores) ([huggingface.co][3])
* **Timing sanity**: too many ultra-short word durations, or big “gaps” in the middle of speech
* **Stability check**: re-run alignment with ±200–400ms padding on segment boundaries; good transcripts stay stable

**Hard filter**: if coverage < X% or score below threshold, drop that transcript *for that utterance*.
This alone often beats naive voting because it nukes hallucinations and language-mismatch outputs.

---

### 2) Do **weighted ROVER / confusion-network voting** (not naive majority)

ROVER is the classic system-combination method for ASR 1-best outputs. ([NIST][6])
Even better: build a **confusion network** and choose words with the best expected WER properties (consensus decoding). ([arXiv][7])

How to make it “insane” instead of average:

* Compute a **weight per system per utterance**:
  (w_j = \mathrm{softmax}(\lambda \cdot \text{alignScore}(T_j) - \mu \cdot \text{unalignedRate}(T_j)))
* In each confusion slot, choose the token maximizing (\sum_j w_j \cdot \mathbf{1}[\text{token}_j=\text{token}])
* If you have per-word confidences from ASR decoders, multiply them in, but **don’t trust them alone** (alignment is your external auditor).

This is directly in the spirit of ROVER (combine multiple ASR outputs to reduce WER) ([NIST][6]) and confusion-network consensus decoding for WER minimization. ([arXiv][7])

---

### 3) Alignment-guided “repair loop” on only the bad spans

After you produce the consensus transcript (T^*):

1. Forced-align (T^*) again.
2. Identify **low-confidence regions**:

* low alignment scores
* clusters of unaligned words
* high disagreement entropy in the confusion network slot

3. For just those spans, re-decode with “expensive mode”:

* shorter audio window (e.g., 2–8s), more context overlap
* higher beam / more temperatures
* for Whisper: prompt with left/right context from (T^*) so it stops freelancing

4. Re-run weighted voting only for that span, splice back into (T^*), then re-align.

This is how you spend compute where it actually buys WER.

---

### 4) The cheat code if you truly need **<1% WER**: allow rejection

If you’re building training data, the easiest way to get **label WER near 0–1%** is not “perfect correction,” it’s **precision-first filtering**:

* Accept only utterances where:

  * consensus alignment coverage is extremely high
  * alignment score is above a strict threshold
  * systems strongly agree (low confusion entropy)

Everything else gets:

* reprocessed (repair loop), or
* routed to human spot-check, or
* dropped

With unlimited data, dropping 20–60% of segments is often cheaper than trying to “fix” them.

---

## Practical recommendations for your language set

* Start with **MMS forced aligner** as the universal validator (Indic scripts + romanization + broad coverage). ([huggingface.co][3])
* Use **WhisperX** if you already run Whisper and want alignment diagnostics + word timestamps quickly (but still keep MMS as independent validator if Whisper is one of your generators). ([GitHub][2])
* Use **MFA** when you’re willing to invest in per-language lexicon/G2P quality (it can be very strong, but the lexicon work matters). ([montreal-forced-aligner.readthedocs.io][1])
* Implement voting as **weighted confusion-network/ROVER**, weighted by alignment quality, not equal votes. ([NIST][6])

## One reality check (important)

At scale, **<1% WER across all utterances** is usually not achievable *purely automatically* unless you (a) constrain the domain hard, and/or (b) **reject** a meaningful portion of data, and/or (c) add some targeted human verification. The pipeline above gets you as close as you can get with compute, and makes it obvious exactly which utterances need escalation.

If you want, I can translate the above into a concrete spec: scoring functions, thresholds to calibrate (with a 1–2 hour hand-labeled dev set per language), and an exact repair-loop schedule.

[1]: https://montreal-forced-aligner.readthedocs.io/en/stable/user_guide/index.html?utm_source=chatgpt.com "User Guide — Montreal Forced Aligner 3.X documentation"
[2]: https://github.com/m-bain/whisperX?utm_source=chatgpt.com "m-bain/whisperX"
[3]: https://huggingface.co/MahmoudAshraf/mms-300m-1130-forced-aligner "MahmoudAshraf/mms-300m-1130-forced-aligner · Hugging Face"
[4]: https://docs.pytorch.org/audio/stable/tutorials/forced_alignment_tutorial.html?utm_source=chatgpt.com "Forced Alignment with Wav2Vec2"
[5]: https://huggingface.co/spaces/Anioji/WhisperX/blob/9c2d684d1a778e559b6b13bb90112712dbc20568/whisperx/alignment.py?utm_source=chatgpt.com "whisperx/alignment.py"
[6]: https://www.nist.gov/publications/post-processing-system-yield-reduced-word-error-rates-recognizer-output-voting-error?utm_source=chatgpt.com "Recognizer Output Voting Error Reduction [ROVER]"
[7]: https://arxiv.org/abs/cs/0010012?utm_source=chatgpt.com "Finding consensus in speech recognition: word error minimization and other applications of confusion networks"


**Planning WER normalization analysis**

Noticed a gap: the current aligner is simplified and doesn't normalize aggressively or use transcripts, likely causing high WER for Gemini. I'll compute alternative normalization (like removing punctuation and digit mapping) to quantify its impact and analyze per-sample WER versus CER to highlight tokenization issues.


**Running analysis on WER-CER differences**

Preparing to identify samples where Gemini's WER notably exceeds CER and consensus WER is high, aiming to isolate problematic cases linked to Gemini or alignment quality.


python3 - <<'PY'
import json
from pathlib import Path

obj = json.loads(Path('indicwhisper_eval_10samples.json').read_text())

rows = []
for lang, samples in obj.items():
    if not isinstance(samples, list):
        continue
    for s in samples:
        rows.append({
            'lang': lang,
            'id': s['sample_id'],
            'ref': s['reference'],
            'gem': s['gemini_hyp'],
            'iw': s['indicwhisper_hyp'],
            'cons': s['consensus_hyp'],
            'gem_wer': s['gemini_wer'],
            'gem_cer': s['gemini_cer'],
            'iw_wer': s['indicwhisper_wer'],
            'cons_wer': s['consensus_wer'],
            'cons_cer': s['consensus_cer'],
        })

# cases where gemini WER high but CER low
candidates = [r for r in rows if r['gem_wer'] > 0.2 and r['gem_cer'] < 0.06]
candidates = sorted(candidates, key=lambda r: (r['gem_wer'] - r['gem_cer']), reverse=True)
print('Gemini high WER but low CER (tokenization/script formatting likely):')
for r in candidates[:6]:
    print('\n', r['lang'], r['id'], 'gem_wer', r['gem_wer'], 'gem_cer', r['gem_cer'])
    print('REF:', r['ref'])
    print('GEM:', r['gem'])

# consensus worse than both models
bad_cons = [r for r in rows if r['cons_wer'] > min(r['gem_wer'], r['iw_wer']) + 1e-9]
bad_cons = sorted(bad_cons, key=lambda r: r['cons_wer'] - min(r['gem_wer'], r['iw_wer']), reverse=True)
print('\nConsensus worse than best model:')
for r in bad_cons[:6]:
    gap = r['cons_wer'] - min(r['gem_wer'], r['iw_wer'])
    print('\n', r['lang'], r['id'], 'cons', r['cons_wer'], 'best', min(r['gem_wer'], r['iw_wer']), 'gap', round(gap,3))
    print('REF:', r['ref'])
    print('CONS:', r['cons'])
    print('GEM :', r['gem'])
    print('IW  :', r['iw'])

PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
Gemini high WER but low CER (tokenization/script formatting likely):

 telugu telugu_000004 gem_wer 0.35714285714285715 gem_cer 0.036036036036036036
REF: పరికరాన్ని గాల్లో కదిలించడం ద్వారా వీడియో గేమ్స్లో యాక్షన్లు మరియు కదలికలను నియంత్రించడానికి ఇది ఆటగాళ్లకు వీలు కల్పిస్తుంది
GEM: పరికరాని గాలిలో కదిలించడం ద్వారా వీడియో గేమ్స్ లో యాక్షన్లు మరియు కదలికలను నియంత్రించడానికి ఇది ఆటగాళ్ళకు వీలు కల్పిస్తుంది.

 telugu telugu_000008 gem_wer 0.2631578947368421 gem_cer 0.021897810218978103
REF: పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడ్డాయి అయితే పైభాగంలో ఇనుప ఖనిజం ఉంటే ట్రాక్ లు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు
GEM: పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడాయి. అయితే పై భాగంలో ఇనుప ఖనిజం ఉంటే ట్రాకులు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు.

Consensus worse than best model:

 telugu telugu_000009 cons 1.1764705882352942 best 0.11764705882352941 gap 1.059
REF: ఇది బేస్ స్టేషన్ లో డ్యూయల్ రేడియోలు ఉంటే 802.11a 802.11b మరియు 802.11gతో అనుకూలంగా వెనుకకు అనుమతిస్తుంది
CONS: ఇది బేస్ స్టేషన్లో డ్యూయల్ రేడియోలు ఉంటే ఎనిమిది వందల రెండు పాయింట్ పదకొండు ఏ ఎనిమిది వందల రెండు పాయింట్ పదకొండు బి మరియు ఎనిమిది వందల రెండు పాయింట్ పదకొండు జితో అనుకూలంగా వెనుకకు అనుమతిస్తుంది
GEM : ఇది బేస్ స్టేషన్ లో డ్యూయల్ రేడియోలు ఉంటే 802.11a, 802.11b మరియు 802.11g తో అనుకూలంగా వెనుకకు అనుమతిస్తుంది.
IW  : ఇది బేస్ స్టేషన్లో డ్యూయల్ రేడియోలు ఉంటే ఎనిమిది వందల రెండు పాయింట్ పదకొండు ఏ ఎనిమిది వందల రెండు పాయింట్ పదకొండు బి మరియు ఎనిమిది వందల రెండు పాయింట్ పదకొండు జితో అనుకూలంగా వెనుకకు అనుమతిస్తుంది

 telugu telugu_000007 cons 0.5 best 0.1 gap 0.4
REF: బుధవారం జరిగిన ఈవెంట్కు ముందు carpanedo ఛాంపియన్షిప్లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు
CONS: బుధవారం జరిగిన ఈవెంట్ కు ముందు కార్పనేడో ఛాంపియన్షిప్ లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు.
GEM : బుధవారం జరిగిన ఈవెంట్ కు ముందు కార్పనేడో ఛాంపియన్షిప్ లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు.
IW  : బుధవారం జరిగిన ఈవెంట్కు ముందు కార్పనేడో ఛాంపియన్షిప్లో రెండు వ్యక్తిగత రేసులలో పాల్గొన్నారు

 hindi hindi_000004 cons 0.2777777777777778 best 0.05555555555555555 gap 0.222
REF: इंटरनेट पर यह खोज शत्रुतापूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी
CONS: इंटरनेट पर यह खोज शत्रुता पूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी। सही है।
GEM : इंटरनेट पर यह खोज शत्रुता पूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी। सही है।
IW  : इंटरनेट पर ये खोज शत्रुतापूर्ण पर्यावरण पाठ्यक्रम के लिए अक्सर आपको एक स्थानीय कंपनी का पता प्रदान करेगी

 telugu telugu_000008 cons 0.2631578947368421 best 0.15789473684210525 gap 0.105
REF: పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడ్డాయి అయితే పైభాగంలో ఇనుప ఖనిజం ఉంటే ట్రాక్ లు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు
CONS: పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడాయి. అయితే పై భాగంలో ఇనుప ఖనిజం ఉంటే ట్రాకులు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు.
GEM : పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడాయి. అయితే పై భాగంలో ఇనుప ఖనిజం ఉంటే ట్రాకులు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు.
IW  : పట్టాలను నిలబెట్టడానికి క్రాస్ టీలు చాలా ముందుగానే ప్రవేశపెట్టబడాయి అయితే పైభాగంలో ఇనుప ఖనిజం ఉంటే ట్రాక్లు మరింత సమర్థవంతంగా ఉంటాయని క్రమంగా గ్రహించారు

 hindi hindi_000006 cons 0.2 best 0.1 gap 0.1
REF: इसे केमिकल का ph कहा जाता है आप लाल गोभी के जूस का इस्तेमाल करके एक संकेतक बना सकते हैं
CONS: इसे केमिकल का पीएच कहा जाता है। आप लाल गोभी के जूस को इस्तेमाल करके एक संकेतक बना सकते हैं।
GEM : इसे केमिकल का पीएच कहा जाता है। आप लाल गोभी के जूस को इस्तेमाल करके एक संकेतक बना सकते हैं।
IW  : इसे केमिकल का पीएच कहा जाता है आप लाल गोभी के जूस को इस्तेमाल करके एक संकेतक बना सकते हैं

 telugu telugu_000002 cons 0.058823529411764705 best 0.0 gap 0.059
REF: మీరు మీ స్వంతం ఆలోచనలతో కాకుండా ప్రభుత్వాల సలహా కూడా తీసుకోవాలని అనుకోవచ్చు అయితే వారి సలహా వారి పౌరుల కోసం రూపొందించబడింది
CONS: మీరు మీ సొంతం ఆలోచనలతో కాకుండా ప్రభుత్వాల సలహా కూడా తీసుకోవాలని అనుకోవచ్చు. అయితే వారి సలహా వారి పౌరుల కోసం రూపొందించబడింది.
GEM : మీరు మీ సొంతం ఆలోచనలతో కాకుండా ప్రభుత్వాల సలహా కూడా తీసుకోవాలని అనుకోవచ్చు. అయితే వారి సలహా వారి పౌరుల కోసం రూపొందించబడింది.
IW  : మీరు మీ స్వంతం ఆలోచనలతో కాకుండా ప్రభుత్వాల సలహా కూడా తీసుకోవాలని అనుకోవచ్చు అయితే వారి సలహా వారి పౌరుల కోసం రూపొందించబడింది


**Analyzing WER issues with Gemini and consensus**

I've identified that high WER often stems from script mismatches or tokenization differences, especially for Indic languages where punctuation and spacing vary; next, I'll explore space-insensitive WER or character-based evaluation to better handle these discrepancies.


**Identifying normalization gaps in evaluation and alignment**

Key sources of WER discrepancies include space variations around suffixes, script mismatches from transliteration, inconsistent number formats, and punctuation differences; improving evaluation fairness likely requires language-specific normalization, placeholder mapping for Latin tokens and numbers, and better alignment using proper forced alignment methods rather than placeholder code.


sed -n '260,560p' transcribe/alignment/mms_aligner.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
        self._model = bundle.get_model()

        if self.device == "cuda":
            import torch
            if torch.cuda.is_available():
                self._model = self._model.to("cuda")

        self._model.eval()
        self._tokenizer = bundle.get_tokenizer()
        self._aligner = bundle.get_aligner()
        self._loaded = True

    def align(
        self,
        audio: np.ndarray,
        sample_rate: int,
        transcript: str,
        language: Optional[str] = None,
    ) -> AlignmentResult:
        """Align using torchaudio forced alignment."""
        if not self._loaded:
            self.load()

        import torch
        import torchaudio

        # Resample if needed
        target_sr = self._bundle.sample_rate
        if sample_rate != target_sr:
            from ..utils.audio import resample_audio
            audio = resample_audio(audio, sample_rate, target_sr)
            sample_rate = target_sr

        # Convert to tensor
        waveform = torch.tensor(audio).unsqueeze(0)
        if self.device == "cuda" and torch.cuda.is_available():
            waveform = waveform.cuda()

        # Get emissions
        with torch.no_grad():
            emissions, _ = self._model(waveform)

        # Tokenize transcript
        tokens = self._tokenizer(transcript)

        # Align
        token_spans = self._aligner(emissions[0], tokens)

        # Convert to our format
        words = transcript.split()
        aligned_words = []

        # Map token spans to words (simplified)
        audio_duration = len(audio) / sample_rate
        n_frames = emissions.shape[1]
        frame_duration = audio_duration / n_frames

        # For now, use simple word-level alignment
        # A proper implementation would map tokens to words
        for i, word in enumerate(words):
            if i < len(token_spans):
                span = token_spans[i]
                start = span.start * frame_duration
                end = span.end * frame_duration
                score = float(span.score) if hasattr(span, 'score') else 0.5
            else:
                # Estimate position
                start = (i / len(words)) * audio_duration
                end = ((i + 1) / len(words)) * audio_duration
                score = 0.1

            aligned_words.append(AlignedWord(
                word=word,
                start=start,
                end=end,
                score=score,
            ))

        # Calculate metrics
        if aligned_words:
            scores = [w.score for w in aligned_words]
            overall_score = np.mean(scores)
            coverage = len([w for w in aligned_words if w.score > 0.3]) / len(aligned_words)
        else:
            overall_score = 0.0
            coverage = 0.0

        return AlignmentResult(
            words=aligned_words,
            overall_score=overall_score,
            coverage=coverage,
            duration=audio_duration,
            avg_word_score=np.mean([w.score for w in aligned_words]) if aligned_words else 0.0,
            min_word_score=min([w.score for w in aligned_words]) if aligned_words else 0.0,
            unaligned_words=[w.word for w in aligned_words if w.score < 0.2],
            timing_issues=self.validate_timing(aligned_words),
        )


**Identifying core alignment issues**

TorchAudioAligner's token-to-word mapping is flawed, leading to potentially incorrect alignments and unreliable scoring, especially since the default aligner ignores tokens in forced alignment. Next, I'll consider alternatives like CTC-based alignment or a simple tokenized Levenshtein similarity to improve transcript scoring.


**Planning minimal alignment fix**

I'll highlight the current alignment scoring as unreliable and may add normalization in consensus to reduce WER. Next, I'll check existing tests to understand expectations and consider implementing proper CTC forced alignment or use torchaudio references if feasible.


sed -n '1,200p' transcribe/tests/test_rover.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
"""Tests for ROVER consensus."""

import pytest
from ..rover import ROVERConsensus, WeightedROVER, ConfusionNetwork

class TestConfusionNetwork:
    """Test confusion network construction."""

    def test_single_transcript(self):
        network = ConfusionNetwork()
        network.add_transcript(["hello", "world"], weight=1.0)
        network.build()

        result = network.get_consensus()
        assert result.text == "hello world"
        assert result.confidence == 1.0

    def test_identical_transcripts(self):
        network = ConfusionNetwork()
        network.add_transcript(["hello", "world"], weight=1.0)
        network.add_transcript(["hello", "world"], weight=1.0)
        network.add_transcript(["hello", "world"], weight=1.0)
        network.build()

        result = network.get_consensus()
        assert result.text == "hello world"
        assert result.agreement_score == 1.0

    def test_majority_vote(self):
        network = ConfusionNetwork()
        network.add_transcript(["hello", "world"], weight=1.0)
        network.add_transcript(["hello", "world"], weight=1.0)
        network.add_transcript(["goodbye", "world"], weight=1.0)
        network.build()

        result = network.get_consensus()
        assert "hello" in result.text  # Majority wins

    def test_weighted_voting(self):
        network = ConfusionNetwork()
        network.add_transcript(["hello"], weight=1.0)
        network.add_transcript(["goodbye"], weight=3.0)  # Higher weight
        network.build()

        result = network.get_consensus()
        assert result.text == "goodbye"  # Higher weight wins

class TestROVERConsensus:
    """Test ROVER consensus algorithm."""

    def test_all_agree(self):
        rover = ROVERConsensus()
        result = rover.combine([
            "hello world",
            "hello world",
            "hello world",
        ])
        assert result.text == "hello world"
        assert result.agreement_score > 0.9

    def test_two_agree(self):
        rover = ROVERConsensus()
        result = rover.combine([
            "hello world",
            "hello world",
            "goodbye world",
        ])
        assert "hello" in result.text

    def test_with_weights(self):
        rover = ROVERConsensus()
        result = rover.combine(
            ["hello", "goodbye", "goodbye"],
            weights=[3.0, 1.0, 1.0]  # hello has higher weight
        )
        assert result.text == "hello"

    def test_empty_input(self):
        rover = ROVERConsensus()
        result = rover.combine([])
        assert result.text == ""
        assert result.confidence == 0.0

class TestWeightedROVER:
    """Test weighted ROVER with alignment scores."""

    def test_alignment_weighted(self):
        rover = WeightedROVER()
        result = rover.combine_with_alignment(
            transcripts=["hello world", "hallo world", "hello word"],
            model_names=["whisper", "indicconformer", "gemini"],
            alignment_scores=[0.9, 0.7, 0.8],
            language="hindi",
        )
        # Higher alignment score should have more influence
        assert "hello" in result.text

    def test_language_specific_weights(self):
        rover = WeightedROVER(base_weights={
            "hindi": (0.5, 0.3, 0.2),  # indicconformer, whisper, gemini
            "english": (0.2, 0.5, 0.3),
        })

        # For Hindi, indicconformer should have more weight
        result = rover.combine_with_alignment(
            transcripts=["नमस्ते", "नमस्ता", "नमस्ते"],
            model_names=["indicconformer", "whisper", "gemini"],
            alignment_scores=[0.8, 0.8, 0.8],
            language="hindi",
        )
        assert "नमस्ते" in result.text  # indicconformer wins due to base weight

class TestDisputedPositions:
    """Test handling of disputed positions."""

    def test_identifies_disputes(self):
        network = ConfusionNetwork()
        network.add_transcript(["hello", "world"], weight=1.0)
        network.add_transcript(["goodbye", "world"], weight=1.0)
        network.add_transcript(["hola", "world"], weight=1.0)
        network.build()

        result = network.get_consensus()
        # First position is disputed
        assert len(result.disputed_positions) > 0 or result.low_agreement_words > 0

    def test_no_disputes_when_agree(self):
        network = ConfusionNetwork()
        network.add_transcript(["hello", "world"], weight=1.0)
        network.add_transcript(["hello", "world"], weight=1.0)
        network.build()

        result = network.get_consensus()
        assert result.low_agreement_words == 0

if __name__ == "__main__":
    pytest.main([__file__, "-v"])


**Planning WER improvements**

I'm considering proposing normalization enhancements to reduce WER, such as updating evaluation functions and adjusting the Gemini prompt to better handle punctuation and alphanumeric codes, while verifying if existing issues or tasks cover this before proceeding.


**Identifying WER normalization and ROVER issues**

I've pinpointed that WER inflation stems from inconsistent number formats and punctuation, suggesting enhancements to normalization by mapping digits and number words and adding looser evaluation metrics like stripped WER or CER; also, ROVER alignment needs normalization improvements, with possible fallback heuristics when alignment is unreliable.


python3 - <<'PY'
import importlib.util
print('torchaudio', importlib.util.find_spec('torchaudio'))
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
torchaudio ModuleSpec(name='torchaudio', loader=<_frozen_importlib_external.SourceFileLoader object at 0x77930ecbd960>, origin='/home/ubuntu/.local/lib/python3.10/site-packages/torchaudio/__init__.py', submodule_search_locations=['/home/ubuntu/.local/lib/python3.10/site-packages/torchaudio'])


**Evaluating TorchAudio for improved alignment**

Torchaudio offers token-level forced alignment that can be mapped to words by grouping character tokens based on their spans and word boundaries marked by special tokens like '|'. Next, I'll inspect the torchaudio tokenizer to confirm tokenization details and plan implementing token-to-word span mapping using its forced alignment utilities.


**Outlining CTC loss scoring to improve alignment**

Implementing full forced alignment is complex, so I'll focus on enhancing the current aligner by computing transcript scores using CTC loss with emissions and tokenized transcripts. This provides meaningful overall alignment scores for better transcript ranking while retaining naive word spans, addressing the consensus pipeline’s main failure cause efficiently.


**Detailing CTC loss computation for alignment scoring**

I’m clarifying how to compute CTC loss-based alignment scores by tokenizing transcripts without special tokens, determining blank token ID from the tokenizer, and calculating loss with proper input and target lengths. The resulting score will be scaled to [0,1] (e.g., using 1/(1+loss)) for meaningful ranking, guiding adjustments to alignment thresholds and overall pipeline scoring.


python3 - <<'PY'
import torchaudio
print('torchaudio', torchaudio.__version__)
from torchaudio.functional import forced_align
print('forced_align', forced_align)
PY


Output:
Exit code: 0
Wall time: 1.8 seconds
Output:
torchaudio 2.9.0+cu128
forced_align <function forced_align at 0x71fad75eb2e0>
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]


Summary: auto


**Reviewing torchaudio forced alignment usage**

The forced aligner returns token spans marked by special `|` tokens for word boundaries; next, I'll verify how the tokenizer handles these and how to group token spans into words by testing with a sample English string.


python3 - <<'PY'
import torchaudio
from torchaudio.pipelines import MMS_FA
bundle = MMS_FA
model = bundle.get_model()
tokenizer = bundle.get_tokenizer()

text = 'hello world'
tokens = tokenizer(text)
print(tokens[:20], len(tokens))
# decode token IDs to string tokens
# tokenizer has get_vocab? maybe returns dictionary
vocab = tokenizer.get_vocab()
inv_vocab = {v:k for k,v in vocab.items()}
print('first tokens decoded:', [inv_vocab[t] for t in tokens[:20]])
PY


Output:
Exit code: 124
Wall time: 10.1 seconds
Output:
command timed out after 10112 milliseconds
Downloading: "https://dl.fbaipublicfiles.com/mms/torchaudio/ctc_alignment_mling_uroman/model.pt" to /home/ubuntu/.cache/torch/hub/checkpoints/model.pt
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]

  0%|          | 0.00/1.18G [00:00<?, ?B/s]
  0%|          | 384k/1.18G [00:00<06:01, 3.49MB/s]
  0%|          | 4.00M/1.18G [00:00<00:55, 22.7MB/s]
  1%|▏         | 17.0M/1.18G [00:00<00:17, 72.8MB/s]
  3%|▎         | 30.2M/1.18G [00:00<00:12, 98.2MB/s]
  4%|▎         | 43.6M/1.18G [00:00<00:10, 113MB/s]
  5%|▍         | 56.9M/1.18G [00:00<00:09, 121MB/s]
  6%|▌         | 70.1M/1.18G [00:00<00:09, 127MB/s]
  7%|▋         | 83.1M/1.18G [00:00<00:09, 129MB/s]
  8%|▊         | 96.6M/1.18G [00:00<00:08, 133MB/s]
  9%|▉         | 110M/1.18G [00:01<00:08, 135MB/s]
 10%|█         | 123M/1.18G [00:01<00:08, 135MB/s]
 11%|█▏        | 136M/1.18G [00:01<00:08, 137MB/s]
 12%|█▏        | 150M/1.18G [00:01<00:08, 137MB/s]
 14%|█▎        | 163M/1.18G [00:01<00:07, 138MB/s]
 15%|█▍        | 176M/1.18G [00:01<00:07, 137MB/s]
 16%|█▌        | 190M/1.18G [00:01<00:07, 139MB/s]
 17%|█▋        | 203M/1.18G [00:01<00:07, 137MB/s]
 18%|█▊        | 217M/1.18G [00:01<00:07, 139MB/s]
 19%|█▉        | 230M/1.18G [00:01<00:07, 138MB/s]
 20%|██        | 244M/1.18G [00:02<00:07, 139MB/s]
 21%|██▏       | 257M/1.18G [00:02<00:07, 138MB/s]
 22%|██▏       | 270M/1.18G [00:02<00:07, 138MB/s]
 24%|██▎       | 284M/1.18G [00:02<00:06, 139MB/s]
 25%|██▍       | 297M/1.18G [00:02<00:06, 138MB/s]
 26%|██▌       | 311M/1.18G [00:02<00:06, 139MB/s]
 27%|██▋       | 324M/1.18G [00:02<00:06, 138MB/s]
 28%|██▊       | 337M/1.18G [00:02<00:06, 138MB/s]
 29%|██▉       | 351M/1.18G [00:02<00:06, 138MB/s]
 30%|███       | 364M/1.18G [00:02<00:06, 138MB/s]
 31%|███▏      | 377M/1.18G [00:03<00:06, 138MB/s]
 32%|███▏      | 390M/1.18G [00:03<00:06, 138MB/s]
 34%|███▎      | 404M/1.18G [00:03<00:06, 139MB/s]
 35%|███▍      | 417M/1.18G [00:03<00:05, 138MB/s]
 36%|███▌      | 431M/1.18G [00:03<00:05, 139MB/s]
 37%|███▋      | 444M/1.18G [00:03<00:05, 139MB/s]
 38%|███▊      | 458M/1.18G [00:03<00:05, 139MB/s]
 39%|███▉      | 471M/1.18G [00:03<00:05, 139MB/s]
 40%|████      | 484M/1.18G [00:03<00:05, 138MB/s]
 41%|████▏     | 497M/1.18G [00:03<00:05, 137MB/s]
 42%|████▏     | 511M/1.18G [00:04<00:05, 136MB/s]
 44%|████▎     | 524M/1.18G [00:04<00:05, 138MB/s]
 45%|████▍     | 538M/1.18G [00:04<00:05, 137MB/s]
 46%|████▌     | 551M/1.18G [00:04<00:04, 139MB/s]
 47%|████▋     | 565M/1.18G [00:04<00:04, 139MB/s]
 48%|████▊     | 578M/1.18G [00:04<00:04, 138MB/s]
 49%|████▉     | 592M/1.18G [00:04<00:04, 140MB/s]
 50%|█████     | 605M/1.18G [00:04<00:04, 138MB/s]
 51%|█████▏    | 618M/1.18G [00:04<00:04, 139MB/s]
 52%|█████▏    | 632M/1.18G [00:04<00:04, 138MB/s]
 54%|█████▎    | 646M/1.18G [00:05<00:04, 139MB/s]
 55%|█████▍    | 659M/1.18G [00:05<00:04, 138MB/s]
 56%|█████▌    | 672M/1.18G [00:05<00:04, 139MB/s]
 57%|█████▋    | 686M/1.18G [00:05<00:03, 138MB/s]
 58%|█████▊    | 699M/1.18G [00:05<00:03, 139MB/s]
 59%|█████▉    | 713M/1.18G [00:05<00:03, 136MB/s]
 60%|██████    | 726M/1.18G [00:05<00:03, 136MB/s]
 61%|██████▏   | 739M/1.18G [00:05<00:03, 138MB/s]
 63%|██████▎   | 753M/1.18G [00:05<00:03, 137MB/s]
 64%|██████▎   | 766M/1.18G [00:05<00:03, 139MB/s]
 65%|██████▍   | 780M/1.18G [00:06<00:03, 137MB/s]
 66%|██████▌   | 793M/1.18G [00:06<00:03, 139MB/s]
 67%|██████▋   | 806M/1.18G [00:06<00:03, 138MB/s]
 68%|██████▊   | 820M/1.18G [00:06<00:02, 139MB/s]
 69%|██████▉   | 833M/1.18G [00:06<00:02, 138MB/s]
 70%|███████   | 847M/1.18G [00:06<00:02, 139MB/s]


**Avoiding large model downloads**

The prior download of a huge model timed out, so it's best to avoid triggering new heavy downloads. I'll focus on reusing the existing models like Wav2Vec2ForCTC to compute alignment scores instead of loading large new ones like torchaudio MMS_FA or "facebook/mms-1b-all."


**Detailing normalization and alignment issues**

I identified key evaluation issues: the current WER normalization misses Indic punctuation and spacing nuances, forced alignment is ineffective causing poor consensus decisions, and tokenization lacks normalization causing vote flipping. I plan to improve normalization for WER and ROVER, fix forced alignment scoring with actual CTC loss, and potentially add heuristics for code-mixed tokens, while proposing new tests and analysis scripts.


Explanation: I’ll triage the high-WER cases, audit the consensus/aligner code, then apply targeted fixes and re-check metrics on the saved results.

Analyze evaluation outputs to identify where Gemini/consensus WER spikes and categorize error types (formatting, tokenization, numbers, hallucinations).

Audit consensus/ROVER and alignment implementations to find scoring/normalization issues driving wrong selections.

Implement fixes: improve normalization for evaluation/ROVER, and make alignment scoring transcript-aware; then re-run a lightweight analysis on saved results to confirm impact.


Output: Plan updated


**Planning nuanced normalization fixes**

I'm outlining updates to unify token normalization for WER and ROVER, aiming to remove Indic punctuation and unify whitespace, optionally lowercase. The plan is to keep original tokens alongside normalized ones for alignment and voting, then return the original token corresponding to the chosen normalized token to preserve output fidelity.


**Analyzing normalization and alignment complexities**

I'm exploring token-level normalization for ROVER to retain original tokens while aligning on normalized forms, noting challenges with tokenization differences especially in Indic scripts. Given the complexity and time, I plan minimal improvements: enhance alignment scoring in forced aligner by computing CTC loss for overall score and improve `normalize_for_wer` to better handle Indic punctuation and whitespace, postponing extensive ROVER normalization for later.


**Implementing CTC-based alignment scoring and enhanced normalization**

I'm specifying integration of CTC loss computation into forced alignment for reliable overall scoring and enhancing WER normalization by removing Unicode punctuation, zero-width characters, and standardizing digits across scripts. The plan includes updating WER tokenization to use this improved normalization for more consistent evaluation.