        'wer_raw': round(sum(result[l]['wer_raw'] for l in lang_keys) / n_langs, 2),
        'wer_norm': round(sum(result[l]['wer_norm'] for l in lang_keys) / n_langs, 2),
        'wer_numcanon': round(sum(result[l]['wer_numcanon'] for l in lang_keys) / n_langs, 2),
        'mer': round(sum(result[l]['mer'] for l in lang_keys) / n_langs, 2),
        'cer_norm': round(sum(result[l]['cer_norm'] for l in lang_keys) / n_langs, 2),

"""
Compute all 4 metric tiers for API baseline models.
Normalization version: v1
"""
import json
import re
import unicodedata
import sys
from collections import defaultdict
from pathlib import Path
from datetime import datetime, timezone

from jiwer import wer as compute_wer, cer as compute_cer

# ── Normalization pipeline ──

# Zero-width characters to strip
ZW_CHARS = re.compile(r'[\u200b-\u200f\u2028-\u202f\ufeff\u00ad]')

# Punctuation sets
# Common punctuation (ASCII + Unicode general)
PUNCT_COMMON = re.compile(r'[!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]')
# Indic punctuation (dandas, etc.) — keep single danda for segmentation? No, remove for norm.
PUNCT_INDIC = re.compile(r'[\u0964\u0965\u0970\u0971]')  # danda, double danda, abbreviation signs
# Extended punctuation
PUNCT_EXTENDED = re.compile(r'[\u2010-\u2027\u2030-\u205e\u2e00-\u2e4f]')
# Quotes normalization
QUOTES = re.compile(r'[\u2018\u2019\u201a\u201b\u201c\u201d\u201e\u201f\u00ab\u00bb]')

# Parenthetical annotations like "(coughing)", "(two seconds pause)"
ANNOTATIONS = re.compile(r'\([^)]*\)')

def norm_raw(text: str) -> str:
    """Step for wer_raw: NFC unicode + trim only."""
    text = unicodedata.normalize('NFC', text)
    text = text.strip()
    return text

def norm_standard(text: str) -> str:
    """Steps for wer_norm: NFKC + strip ZW + whitespace + punct + case fold."""
    # Step 1: NFKC
    text = unicodedata.normalize('NFKC', text)
    # Step 2: Strip zero-width chars
    text = ZW_CHARS.sub('', text)
    # Step 2.5: Remove parenthetical annotations
    text = ANNOTATIONS.sub('', text)
    # Step 3: Normalize whitespace
    text = ' '.join(text.split())
    # Step 4: Standardize quotes/dashes
    text = QUOTES.sub("'", text)
    text = text.replace('\u2014', '-').replace('\u2013', '-').replace('\u2012', '-')
    # Step 5: Remove punctuation
    text = PUNCT_COMMON.sub('', text)
    text = PUNCT_INDIC.sub('', text)
    text = PUNCT_EXTENDED.sub('', text)
    # Step 6: Case fold (affects English only, Indic scripts have no case)
    text = text.lower()
    # Final whitespace cleanup
    text = ' '.join(text.split()).strip()
    return text

def norm_mer(text: str) -> str:
    """Steps for MER: norm_standard + remove ALL spaces."""
    return norm_standard(text).replace(' ', '')

def norm_numcanon(text: str) -> str:
    """Steps for wer_numcanon: norm_standard + number canonicalization."""
    text = norm_standard(text)
    # Remove commas in numbers (e.g., "25,000" -> "25000")
    text = re.sub(r'(\d),(\d)', r'\1\2', text)
    # Normalize Devanagari digits to ASCII
    for offset, base in [(0x0966, '0'), (0x09E6, '0'), (0x0A66, '0'),
                          (0x0AE6, '0'), (0x0B66, '0'), (0x0BE6, '0'),
                          (0x0C66, '0'), (0x0CE6, '0'), (0x0D66, '0')]:
        for i in range(10):
            text = text.replace(chr(offset + i), str(i))
    return text

# ── Metric computation ──

def compute_metrics_for_samples(samples: list[dict]) -> dict:
    """Compute all 4 metric tiers for a list of samples."""
    by_lang = defaultdict(list)
    for s in samples:
        by_lang[s['language']].append(s)

    result = {}
    all_ref_raw, all_hyp_raw = [], []
    all_ref_norm, all_hyp_norm = [], []
    all_ref_nc, all_hyp_nc = [], []
    all_ref_mer, all_hyp_mer = [], []

    for lang in sorted(by_lang.keys()):
        lang_samples = by_lang[lang]
        refs_raw, hyps_raw = [], []
        refs_norm, hyps_norm = [], []
        refs_nc, hyps_nc = [], []
        refs_mer, hyps_mer = [], []
        empty_count = 0

        for s in lang_samples:
            ref = s['reference']
            hyp = s.get('transcript', '') or ''

            if not hyp.strip():
                empty_count += 1

            rr, hr = norm_raw(ref), norm_raw(hyp)
            rn, hn = norm_standard(ref), norm_standard(hyp)
            rnc, hnc = norm_numcanon(ref), norm_numcanon(hyp)
            rm, hm = norm_mer(ref), norm_mer(hyp)

            # Skip empty refs (shouldn't happen but safety)
            if not rr.strip():
                continue

            refs_raw.append(rr); hyps_raw.append(hr if hr.strip() else '<empty>')
            refs_norm.append(rn if rn else '<empty>'); hyps_norm.append(hn if hn else '<empty>')
            refs_nc.append(rnc if rnc else '<empty>'); hyps_nc.append(hnc if hnc else '<empty>')
            refs_mer.append(rm if rm else '<empty>'); hyps_mer.append(hm if hm else '<empty>')

        w_raw = compute_wer(refs_raw, hyps_raw) * 100
        w_norm = compute_wer(refs_norm, hyps_norm) * 100
        w_nc = compute_wer(refs_nc, hyps_nc) * 100
        c_norm = compute_cer(refs_norm, hyps_norm) * 100
        # MER: CER on space-stripped text (each sample is one "word")
        m_er = compute_cer(refs_mer, hyps_mer) * 100

        result[lang] = {
            'n_samples': len(lang_samples),
            'wer_raw': round(w_raw, 2),
            'wer_norm': round(w_norm, 2),
            'wer_numcanon': round(w_nc, 2),
            'mer': round(m_er, 2),
            'cer_norm': round(c_norm, 2),
            'empty_hypotheses': empty_count,
            'normalization_delta': {
                'raw_to_norm': round(w_raw - w_norm, 2),
                'norm_to_numcanon': round(w_norm - w_nc, 2),
                'norm_to_mer': round(w_norm - m_er, 2),
            }
        }

        all_ref_raw.extend(refs_raw); all_hyp_raw.extend(hyps_raw)
        all_ref_norm.extend(refs_norm); all_hyp_norm.extend(hyps_norm)
        all_ref_nc.extend(refs_nc); all_hyp_nc.extend(hyps_nc)
        all_ref_mer.extend(refs_mer); all_hyp_mer.extend(hyps_mer)

    # Overall (micro-average)
    result['__overall__'] = {
        'n_samples': len(samples),
        'wer_raw': round(compute_wer(all_ref_raw, all_hyp_raw) * 100, 2),
        'wer_norm': round(compute_wer(all_ref_norm, all_hyp_norm) * 100, 2),
        'wer_numcanon': round(compute_wer(all_ref_nc, all_hyp_nc) * 100, 2),
        'mer': round(compute_cer(all_ref_mer, all_hyp_mer) * 100, 2),
        'cer_norm': round(compute_cer(all_ref_norm, all_hyp_norm) * 100, 2),
    }

    # Macro average
    lang_keys = [k for k in result if not k.startswith('_')]
    n_langs = len(lang_keys)
    result['__macro_avg__'] = {
        'n_languages': n_langs,
        'wer_raw': round(sum(result[l]['wer_raw'] for l in lang_keys) / n_langs, 2),
        'wer_norm': round(sum(result[l]['wer_norm'] for l in lang_keys) / n_langs, 2),
        'wer_numcanon': round(sum(result[l]['wer_numcanon'] for l in lang_keys) / n_langs, 2),
        'cer_norm': round(sum(result[l]['cer_norm'] for l in lang_keys) / n_langs, 2),
    }

    return result

def build_sample_analysis(samples: list[dict]) -> list[dict]:
    """Build sample_analysis.json entries."""
    out = []
    for s in samples:
        ref = s['reference']
        hyp = s.get('transcript', '') or ''
        rn = norm_standard(ref)
        hn = norm_standard(hyp)
        rnc = norm_numcanon(ref)
        hnc = norm_numcanon(hyp)

        flags = []
        if ref == hyp:
            flags.append('exact_match')
        if rn == hn:
            flags.append('exact_match_norm')
        if ref != hyp and rn == hn:
            flags.append('punctuation_only_diff')
        if not hyp.strip():
            flags.append('empty_hypothesis')
        # Numeric mismatch: digits present in either
        if re.search(r'\d', ref) or re.search(r'\d', hyp):
            if rn != hn:
                flags.append('numeric_mismatch')

        out.append({
            'id': s['id'],
            'language': s['language'],
            'reference': ref,
            'hypothesis': hyp,
            'ref_norm': rn,
            'hyp_norm': hn,
            'ref_numcanon': rnc,
            'hyp_numcanon': hnc,
            'detected_language': s.get('detected_lang', ''),
            'flags': flags,
        })
    return out

# ── Model registry ──

MODELS = {
    'elevenlabs_full_6k.jsonl': {
        'model_id': 'elevenlabs-scribe-v2',
        'display_name': 'ElevenLabs Scribe v2',
        'model_type': 'elevenlabs-scribe-v2',
    },
    'elevenlabs_v1_full_6k.jsonl': {
        'model_id': 'elevenlabs-scribe-v1',
        'display_name': 'ElevenLabs Scribe v1',
        'model_type': 'elevenlabs-scribe-v1',
    },
    'gemini_3_flash_strict_full_6k.jsonl': {
        'model_id': 'gemini-3-flash-strict',
        'display_name': 'Gemini 3 Flash (Strict)',
        'model_type': 'gemini-3-flash-preview',
    },
    'gemini_gemini_3_flash_preview_full_6k.jsonl': {
        'model_id': 'gemini-3-flash-preview',
        'display_name': 'Gemini 3 Flash (Preview)',
        'model_type': 'gemini-3-flash-preview',
    },
    'gemini_flash_v2_full_6k.jsonl': {
        'model_id': 'gemini-2.5-flash',
        'display_name': 'Gemini 2.5 Flash',
        'model_type': 'gemini-2.5-flash',
    },
    'gemini_gemini_3_1_pro_preview_full_6k.jsonl': {
        'model_id': 'gemini-3.1-pro',
        'display_name': 'Gemini 3.1 Pro',
        'model_type': 'gemini-3.1-pro-preview',
    },
    'sarvam_full_6k.jsonl': {
        'model_id': 'sarvam-saaras-v3',
        'display_name': 'Sarvam Saaras v3',
        'model_type': 'saaras:v3',
    },
}

OUTPUT_BASE = Path('/home/ubuntu/training/benchmark_outputs')

def process_model(filename: str, info: dict):
    model_id = info['model_id']
    print(f'\n{"="*60}')
    print(f'Processing: {info["display_name"]} ({filename})')
    print(f'{"="*60}')

    with open(filename) as f:
        samples = [json.loads(line) for line in f]

    print(f'  Loaded {len(samples)} samples')

    # Compute metrics
    metrics = compute_metrics_for_samples(samples)

    # Add meta
    total_audio = sum(s.get('duration', 0) for s in samples)
    metrics['__meta__'] = {
        'checkpoint': f'api/{model_id}',
        'checkpoint_name': 'baseline',
        'model_id': model_id,
        'model_type': info['model_type'],
        'dataset': 'BayAreaBoys/indic-asr-benchmark-6k',
        'batch_size': 1,
        'inference_time_sec': 0,
        'total_audio_sec': round(total_audio, 2),
        'rtf': 0,
        'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
        'normalization_version': 'v1',
        'framework': 'api',
    }

    # Print summary
    o = metrics['__overall__']
    print(f'  Overall: wer_raw={o["wer_raw"]:.2f}  wer_norm={o["wer_norm"]:.2f}  wer_numcanon={o["wer_numcanon"]:.2f}  cer_norm={o["cer_norm"]:.2f}')

    # Build sample analysis
    sample_analysis = build_sample_analysis(samples)

    # Write outputs
    out_dir = OUTPUT_BASE / model_id / 'baseline'
    out_dir.mkdir(parents=True, exist_ok=True)

    with open(out_dir / 'metrics.json', 'w') as f:
        json.dump(metrics, f, indent=2, ensure_ascii=False)

    with open(out_dir / 'sample_analysis.json', 'w') as f:
        json.dump(sample_analysis, f, indent=2, ensure_ascii=False)

    print(f'  Written to {out_dir}/')
    return metrics

if __name__ == '__main__':
    results = {}
    for filename, info in MODELS.items():
        fpath = Path(filename)
        if not fpath.exists():
            fpath = Path('/home/ubuntu/training/api_results') / filename
        if not fpath.exists():
            print(f'SKIP: {filename} not found')
            continue
        results[info['model_id']] = process_model(str(fpath), info)

    # Print comparison table
    print(f'\n\n{"="*80}')
    print('COMPARISON TABLE (all models)')
    print(f'{"="*80}')
    print(f'{"Model":<30} {"WER Raw":>8} {"WER Norm":>9} {"WER Num":>8} {"CER Norm":>9}')
    print('-' * 70)
    for mid, m in sorted(results.items(), key=lambda x: x[1]['__overall__']['wer_norm']):
        o = m['__overall__']
        print(f'{mid:<30} {o["wer_raw"]:>7.2f}% {o["wer_norm"]:>8.2f}% {o["wer_numcanon"]:>7.2f}% {o["cer_norm"]:>8.2f}%')

+        'mer': round(sum(result[l]['mer'] for l in lang_keys) / n_langs, 2),


Also update build_sample_analysis to include MER fields:


Edit

        rn = norm_standard(ref)
        hn = norm_standard(hyp)
        rnc = norm_numcanon(ref)
        hnc = norm_numcanon(hyp)

        rn = norm_standard(ref)
        hn = norm_standard(hyp)
        rnc = norm_numcanon(ref)
        hnc = norm_numcanon(hyp)
        rm = norm_mer(ref)
        hm = norm_mer(hyp)


The file /home/ubuntu/training/api_results/compute_metrics.py has been updated successfully.

        rn = norm_standard(ref)
        hn = norm_standard(hyp)
        rnc = norm_numcanon(ref)
        hnc = norm_numcanon(hyp)
        rm = norm_mer(ref)
        hm = norm_mer(hyp)

"""
Compute all 4 metric tiers for API baseline models.
Normalization version: v1
"""
import json
import re
import unicodedata
import sys
from collections import defaultdict
from pathlib import Path
from datetime import datetime, timezone

from jiwer import wer as compute_wer, cer as compute_cer

# ── Normalization pipeline ──

# Zero-width characters to strip
ZW_CHARS = re.compile(r'[\u200b-\u200f\u2028-\u202f\ufeff\u00ad]')

# Punctuation sets
# Common punctuation (ASCII + Unicode general)
PUNCT_COMMON = re.compile(r'[!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]')
# Indic punctuation (dandas, etc.) — keep single danda for segmentation? No, remove for norm.
PUNCT_INDIC = re.compile(r'[\u0964\u0965\u0970\u0971]')  # danda, double danda, abbreviation signs
# Extended punctuation
PUNCT_EXTENDED = re.compile(r'[\u2010-\u2027\u2030-\u205e\u2e00-\u2e4f]')
# Quotes normalization
QUOTES = re.compile(r'[\u2018\u2019\u201a\u201b\u201c\u201d\u201e\u201f\u00ab\u00bb]')

# Parenthetical annotations like "(coughing)", "(two seconds pause)"
ANNOTATIONS = re.compile(r'\([^)]*\)')

def norm_raw(text: str) -> str:
    """Step for wer_raw: NFC unicode + trim only."""
    text = unicodedata.normalize('NFC', text)
    text = text.strip()
    return text

def norm_standard(text: str) -> str:
    """Steps for wer_norm: NFKC + strip ZW + whitespace + punct + case fold."""
    # Step 1: NFKC
    text = unicodedata.normalize('NFKC', text)
    # Step 2: Strip zero-width chars
    text = ZW_CHARS.sub('', text)
    # Step 2.5: Remove parenthetical annotations
    text = ANNOTATIONS.sub('', text)
    # Step 3: Normalize whitespace
    text = ' '.join(text.split())
    # Step 4: Standardize quotes/dashes
    text = QUOTES.sub("'", text)
    text = text.replace('\u2014', '-').replace('\u2013', '-').replace('\u2012', '-')
    # Step 5: Remove punctuation
    text = PUNCT_COMMON.sub('', text)
    text = PUNCT_INDIC.sub('', text)
    text = PUNCT_EXTENDED.sub('', text)
    # Step 6: Case fold (affects English only, Indic scripts have no case)
    text = text.lower()
    # Final whitespace cleanup
    text = ' '.join(text.split()).strip()
    return text

def norm_mer(text: str) -> str:
    """Steps for MER: norm_standard + remove ALL spaces."""
    return norm_standard(text).replace(' ', '')

def norm_numcanon(text: str) -> str:
    """Steps for wer_numcanon: norm_standard + number canonicalization."""
    text = norm_standard(text)
    # Remove commas in numbers (e.g., "25,000" -> "25000")
    text = re.sub(r'(\d),(\d)', r'\1\2', text)
    # Normalize Devanagari digits to ASCII
    for offset, base in [(0x0966, '0'), (0x09E6, '0'), (0x0A66, '0'),
                          (0x0AE6, '0'), (0x0B66, '0'), (0x0BE6, '0'),
                          (0x0C66, '0'), (0x0CE6, '0'), (0x0D66, '0')]:
        for i in range(10):
            text = text.replace(chr(offset + i), str(i))
    return text

# ── Metric computation ──

def compute_metrics_for_samples(samples: list[dict]) -> dict:
    """Compute all 4 metric tiers for a list of samples."""
    by_lang = defaultdict(list)
    for s in samples:
        by_lang[s['language']].append(s)

    result = {}
    all_ref_raw, all_hyp_raw = [], []
    all_ref_norm, all_hyp_norm = [], []
    all_ref_nc, all_hyp_nc = [], []
    all_ref_mer, all_hyp_mer = [], []

    for lang in sorted(by_lang.keys()):
        lang_samples = by_lang[lang]
        refs_raw, hyps_raw = [], []
        refs_norm, hyps_norm = [], []
        refs_nc, hyps_nc = [], []
        refs_mer, hyps_mer = [], []
        empty_count = 0

        for s in lang_samples:
            ref = s['reference']
            hyp = s.get('transcript', '') or ''

            if not hyp.strip():
                empty_count += 1

            rr, hr = norm_raw(ref), norm_raw(hyp)
            rn, hn = norm_standard(ref), norm_standard(hyp)
            rnc, hnc = norm_numcanon(ref), norm_numcanon(hyp)
            rm, hm = norm_mer(ref), norm_mer(hyp)

            # Skip empty refs (shouldn't happen but safety)
            if not rr.strip():
                continue

            refs_raw.append(rr); hyps_raw.append(hr if hr.strip() else '<empty>')
            refs_norm.append(rn if rn else '<empty>'); hyps_norm.append(hn if hn else '<empty>')
            refs_nc.append(rnc if rnc else '<empty>'); hyps_nc.append(hnc if hnc else '<empty>')
            refs_mer.append(rm if rm else '<empty>'); hyps_mer.append(hm if hm else '<empty>')

        w_raw = compute_wer(refs_raw, hyps_raw) * 100
        w_norm = compute_wer(refs_norm, hyps_norm) * 100
        w_nc = compute_wer(refs_nc, hyps_nc) * 100
        c_norm = compute_cer(refs_norm, hyps_norm) * 100
        # MER: CER on space-stripped text (each sample is one "word")
        m_er = compute_cer(refs_mer, hyps_mer) * 100

        result[lang] = {
            'n_samples': len(lang_samples),
            'wer_raw': round(w_raw, 2),
            'wer_norm': round(w_norm, 2),
            'wer_numcanon': round(w_nc, 2),
            'mer': round(m_er, 2),
            'cer_norm': round(c_norm, 2),
            'empty_hypotheses': empty_count,
            'normalization_delta': {
                'raw_to_norm': round(w_raw - w_norm, 2),
                'norm_to_numcanon': round(w_norm - w_nc, 2),
                'norm_to_mer': round(w_norm - m_er, 2),
            }
        }

        all_ref_raw.extend(refs_raw); all_hyp_raw.extend(hyps_raw)
        all_ref_norm.extend(refs_norm); all_hyp_norm.extend(hyps_norm)
        all_ref_nc.extend(refs_nc); all_hyp_nc.extend(hyps_nc)
        all_ref_mer.extend(refs_mer); all_hyp_mer.extend(hyps_mer)

    # Overall (micro-average)
    result['__overall__'] = {
        'n_samples': len(samples),
        'wer_raw': round(compute_wer(all_ref_raw, all_hyp_raw) * 100, 2),
        'wer_norm': round(compute_wer(all_ref_norm, all_hyp_norm) * 100, 2),
        'wer_numcanon': round(compute_wer(all_ref_nc, all_hyp_nc) * 100, 2),
        'mer': round(compute_cer(all_ref_mer, all_hyp_mer) * 100, 2),
        'cer_norm': round(compute_cer(all_ref_norm, all_hyp_norm) * 100, 2),
    }

    # Macro average
    lang_keys = [k for k in result if not k.startswith('_')]
    n_langs = len(lang_keys)
    result['__macro_avg__'] = {
        'n_languages': n_langs,
        'wer_raw': round(sum(result[l]['wer_raw'] for l in lang_keys) / n_langs, 2),
        'wer_norm': round(sum(result[l]['wer_norm'] for l in lang_keys) / n_langs, 2),
        'wer_numcanon': round(sum(result[l]['wer_numcanon'] for l in lang_keys) / n_langs, 2),
        'mer': round(sum(result[l]['mer'] for l in lang_keys) / n_langs, 2),
        'cer_norm': round(sum(result[l]['cer_norm'] for l in lang_keys) / n_langs, 2),
    }

    return result

def build_sample_analysis(samples: list[dict]) -> list[dict]:
    """Build sample_analysis.json entries."""
    out = []
    for s in samples:
        ref = s['reference']
        hyp = s.get('transcript', '') or ''
        rn = norm_standard(ref)
        hn = norm_standard(hyp)
        rnc = norm_numcanon(ref)
        hnc = norm_numcanon(hyp)

        flags = []
        if ref == hyp:
            flags.append('exact_match')
        if rn == hn:
            flags.append('exact_match_norm')
        if ref != hyp and rn == hn:
            flags.append('punctuation_only_diff')
        if not hyp.strip():
            flags.append('empty_hypothesis')
        # Numeric mismatch: digits present in either
        if re.search(r'\d', ref) or re.search(r'\d', hyp):
            if rn != hn:
                flags.append('numeric_mismatch')

        out.append({
            'id': s['id'],
            'language': s['language'],
            'reference': ref,
            'hypothesis': hyp,
            'ref_norm': rn,
            'hyp_norm': hn,
            'ref_numcanon': rnc,
            'hyp_numcanon': hnc,
            'detected_language': s.get('detected_lang', ''),
            'flags': flags,
        })
    return out

# ── Model registry ──

MODELS = {
    'elevenlabs_full_6k.jsonl': {
        'model_id': 'elevenlabs-scribe-v2',
        'display_name': 'ElevenLabs Scribe v2',
        'model_type': 'elevenlabs-scribe-v2',
    },
    'elevenlabs_v1_full_6k.jsonl': {
        'model_id': 'elevenlabs-scribe-v1',
        'display_name': 'ElevenLabs Scribe v1',
        'model_type': 'elevenlabs-scribe-v1',
    },
    'gemini_3_flash_strict_full_6k.jsonl': {
        'model_id': 'gemini-3-flash-strict',
        'display_name': 'Gemini 3 Flash (Strict)',
        'model_type': 'gemini-3-flash-preview',
    },
    'gemini_gemini_3_flash_preview_full_6k.jsonl': {
        'model_id': 'gemini-3-flash-preview',
        'display_name': 'Gemini 3 Flash (Preview)',
        'model_type': 'gemini-3-flash-preview',
    },
    'gemini_flash_v2_full_6k.jsonl': {
        'model_id': 'gemini-2.5-flash',
        'display_name': 'Gemini 2.5 Flash',
        'model_type': 'gemini-2.5-flash',
    },
    'gemini_gemini_3_1_pro_preview_full_6k.jsonl': {
        'model_id': 'gemini-3.1-pro',
        'display_name': 'Gemini 3.1 Pro',
        'model_type': 'gemini-3.1-pro-preview',
    },
    'sarvam_full_6k.jsonl': {
        'model_id': 'sarvam-saaras-v3',
        'display_name': 'Sarvam Saaras v3',
        'model_type': 'saaras:v3',
    },
}

OUTPUT_BASE = Path('/home/ubuntu/training/benchmark_outputs')

def process_model(filename: str, info: dict):
    model_id = info['model_id']
    print(f'\n{"="*60}')
    print(f'Processing: {info["display_name"]} ({filename})')
    print(f'{"="*60}')

    with open(filename) as f:
        samples = [json.loads(line) for line in f]

    print(f'  Loaded {len(samples)} samples')

    # Compute metrics
    metrics = compute_metrics_for_samples(samples)

    # Add meta
    total_audio = sum(s.get('duration', 0) for s in samples)
    metrics['__meta__'] = {
        'checkpoint': f'api/{model_id}',
        'checkpoint_name': 'baseline',
        'model_id': model_id,
        'model_type': info['model_type'],
        'dataset': 'BayAreaBoys/indic-asr-benchmark-6k',
        'batch_size': 1,
        'inference_time_sec': 0,
        'total_audio_sec': round(total_audio, 2),
        'rtf': 0,
        'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
        'normalization_version': 'v1',
        'framework': 'api',
    }

    # Print summary
    o = metrics['__overall__']
    print(f'  Overall: wer_raw={o["wer_raw"]:.2f}  wer_norm={o["wer_norm"]:.2f}  wer_numcanon={o["wer_numcanon"]:.2f}  cer_norm={o["cer_norm"]:.2f}')

    # Build sample analysis
    sample_analysis = build_sample_analysis(samples)

    # Write outputs
    out_dir = OUTPUT_BASE / model_id / 'baseline'
    out_dir.mkdir(parents=True, exist_ok=True)

    with open(out_dir / 'metrics.json', 'w') as f:
        json.dump(metrics, f, indent=2, ensure_ascii=False)

    with open(out_dir / 'sample_analysis.json', 'w') as f:
        json.dump(sample_analysis, f, indent=2, ensure_ascii=False)

    print(f'  Written to {out_dir}/')
    return metrics

if __name__ == '__main__':
    results = {}
    for filename, info in MODELS.items():
        fpath = Path(filename)
        if not fpath.exists():
            fpath = Path('/home/ubuntu/training/api_results') / filename
        if not fpath.exists():
            print(f'SKIP: {filename} not found')
            continue
        results[info['model_id']] = process_model(str(fpath), info)

    # Print comparison table
    print(f'\n\n{"="*80}')
    print('COMPARISON TABLE (all models)')
    print(f'{"="*80}')
    print(f'{"Model":<30} {"WER Raw":>8} {"WER Norm":>9} {"WER Num":>8} {"CER Norm":>9}')
    print('-' * 70)
    for mid, m in sorted(results.items(), key=lambda x: x[1]['__overall__']['wer_norm']):
        o = m['__overall__']
        print(f'{mid:<30} {o["wer_raw"]:>7.2f}% {o["wer_norm"]:>8.2f}% {o["wer_numcanon"]:>7.2f}% {o["cer_norm"]:>8.2f}%')

hn = norm_standard(hyp)
rnc = norm_numcanon(ref)
hnc = norm_numcanon(hyp)
+        rm = norm_mer(ref)
+        hm = norm_mer(hyp)
flags = []
if ref == hyp:


            'ref_numcanon': rnc,
            'hyp_numcanon': hnc,

            'ref_numcanon': rnc,
            'hyp_numcanon': hnc,
            'ref_mer': rm,
            'hyp_mer': hm,


            'ref_numcanon': rnc,
            'hyp_numcanon': hnc,
            'ref_mer': rm,
            'hyp_mer': hm,

"""
Compute all 4 metric tiers for API baseline models.
Normalization version: v1
"""
import json
import re
import unicodedata
import sys
from collections import defaultdict
from pathlib import Path
from datetime import datetime, timezone

from jiwer import wer as compute_wer, cer as compute_cer

# ── Normalization pipeline ──

# Zero-width characters to strip
ZW_CHARS = re.compile(r'[\u200b-\u200f\u2028-\u202f\ufeff\u00ad]')

# Punctuation sets
# Common punctuation (ASCII + Unicode general)
PUNCT_COMMON = re.compile(r'[!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]')
# Indic punctuation (dandas, etc.) — keep single danda for segmentation? No, remove for norm.
PUNCT_INDIC = re.compile(r'[\u0964\u0965\u0970\u0971]')  # danda, double danda, abbreviation signs
# Extended punctuation
PUNCT_EXTENDED = re.compile(r'[\u2010-\u2027\u2030-\u205e\u2e00-\u2e4f]')
# Quotes normalization
QUOTES = re.compile(r'[\u2018\u2019\u201a\u201b\u201c\u201d\u201e\u201f\u00ab\u00bb]')

# Parenthetical annotations like "(coughing)", "(two seconds pause)"
ANNOTATIONS = re.compile(r'\([^)]*\)')

def norm_raw(text: str) -> str:
    """Step for wer_raw: NFC unicode + trim only."""
    text = unicodedata.normalize('NFC', text)
    text = text.strip()
    return text

def norm_standard(text: str) -> str:
    """Steps for wer_norm: NFKC + strip ZW + whitespace + punct + case fold."""
    # Step 1: NFKC
    text = unicodedata.normalize('NFKC', text)
    # Step 2: Strip zero-width chars
    text = ZW_CHARS.sub('', text)
    # Step 2.5: Remove parenthetical annotations
    text = ANNOTATIONS.sub('', text)
    # Step 3: Normalize whitespace
    text = ' '.join(text.split())
    # Step 4: Standardize quotes/dashes
    text = QUOTES.sub("'", text)
    text = text.replace('\u2014', '-').replace('\u2013', '-').replace('\u2012', '-')
    # Step 5: Remove punctuation
    text = PUNCT_COMMON.sub('', text)
    text = PUNCT_INDIC.sub('', text)
    text = PUNCT_EXTENDED.sub('', text)
    # Step 6: Case fold (affects English only, Indic scripts have no case)
    text = text.lower()
    # Final whitespace cleanup
    text = ' '.join(text.split()).strip()
    return text

def norm_mer(text: str) -> str:
    """Steps for MER: norm_standard + remove ALL spaces."""
    return norm_standard(text).replace(' ', '')

def norm_numcanon(text: str) -> str:
    """Steps for wer_numcanon: norm_standard + number canonicalization."""
    text = norm_standard(text)
    # Remove commas in numbers (e.g., "25,000" -> "25000")
    text = re.sub(r'(\d),(\d)', r'\1\2', text)
    # Normalize Devanagari digits to ASCII
    for offset, base in [(0x0966, '0'), (0x09E6, '0'), (0x0A66, '0'),
                          (0x0AE6, '0'), (0x0B66, '0'), (0x0BE6, '0'),
                          (0x0C66, '0'), (0x0CE6, '0'), (0x0D66, '0')]:
        for i in range(10):
            text = text.replace(chr(offset + i), str(i))
    return text

# ── Metric computation ──

def compute_metrics_for_samples(samples: list[dict]) -> dict:
    """Compute all 4 metric tiers for a list of samples."""
    by_lang = defaultdict(list)
    for s in samples:
        by_lang[s['language']].append(s)

    result = {}
    all_ref_raw, all_hyp_raw = [], []
    all_ref_norm, all_hyp_norm = [], []
    all_ref_nc, all_hyp_nc = [], []
    all_ref_mer, all_hyp_mer = [], []

    for lang in sorted(by_lang.keys()):
        lang_samples = by_lang[lang]
        refs_raw, hyps_raw = [], []
        refs_norm, hyps_norm = [], []
        refs_nc, hyps_nc = [], []
        refs_mer, hyps_mer = [], []
        empty_count = 0

        for s in lang_samples:
            ref = s['reference']
            hyp = s.get('transcript', '') or ''

            if not hyp.strip():
                empty_count += 1

            rr, hr = norm_raw(ref), norm_raw(hyp)
            rn, hn = norm_standard(ref), norm_standard(hyp)
            rnc, hnc = norm_numcanon(ref), norm_numcanon(hyp)
            rm, hm = norm_mer(ref), norm_mer(hyp)

            # Skip empty refs (shouldn't happen but safety)
            if not rr.strip():
                continue

            refs_raw.append(rr); hyps_raw.append(hr if hr.strip() else '<empty>')
            refs_norm.append(rn if rn else '<empty>'); hyps_norm.append(hn if hn else '<empty>')
            refs_nc.append(rnc if rnc else '<empty>'); hyps_nc.append(hnc if hnc else '<empty>')
            refs_mer.append(rm if rm else '<empty>'); hyps_mer.append(hm if hm else '<empty>')

        w_raw = compute_wer(refs_raw, hyps_raw) * 100
        w_norm = compute_wer(refs_norm, hyps_norm) * 100
        w_nc = compute_wer(refs_nc, hyps_nc) * 100
        c_norm = compute_cer(refs_norm, hyps_norm) * 100
        # MER: CER on space-stripped text (each sample is one "word")
        m_er = compute_cer(refs_mer, hyps_mer) * 100

        result[lang] = {
            'n_samples': len(lang_samples),
            'wer_raw': round(w_raw, 2),
            'wer_norm': round(w_norm, 2),
            'wer_numcanon': round(w_nc, 2),
            'mer': round(m_er, 2),
            'cer_norm': round(c_norm, 2),
            'empty_hypotheses': empty_count,
            'normalization_delta': {
                'raw_to_norm': round(w_raw - w_norm, 2),
                'norm_to_numcanon': round(w_norm - w_nc, 2),
                'norm_to_mer': round(w_norm - m_er, 2),
            }
        }

        all_ref_raw.extend(refs_raw); all_hyp_raw.extend(hyps_raw)
        all_ref_norm.extend(refs_norm); all_hyp_norm.extend(hyps_norm)
        all_ref_nc.extend(refs_nc); all_hyp_nc.extend(hyps_nc)
        all_ref_mer.extend(refs_mer); all_hyp_mer.extend(hyps_mer)

    # Overall (micro-average)
    result['__overall__'] = {
        'n_samples': len(samples),
        'wer_raw': round(compute_wer(all_ref_raw, all_hyp_raw) * 100, 2),
        'wer_norm': round(compute_wer(all_ref_norm, all_hyp_norm) * 100, 2),
        'wer_numcanon': round(compute_wer(all_ref_nc, all_hyp_nc) * 100, 2),
        'mer': round(compute_cer(all_ref_mer, all_hyp_mer) * 100, 2),
        'cer_norm': round(compute_cer(all_ref_norm, all_hyp_norm) * 100, 2),
    }

    # Macro average
    lang_keys = [k for k in result if not k.startswith('_')]
    n_langs = len(lang_keys)
    result['__macro_avg__'] = {
        'n_languages': n_langs,
        'wer_raw': round(sum(result[l]['wer_raw'] for l in lang_keys) / n_langs, 2),
        'wer_norm': round(sum(result[l]['wer_norm'] for l in lang_keys) / n_langs, 2),
        'wer_numcanon': round(sum(result[l]['wer_numcanon'] for l in lang_keys) / n_langs, 2),
        'mer': round(sum(result[l]['mer'] for l in lang_keys) / n_langs, 2),
        'cer_norm': round(sum(result[l]['cer_norm'] for l in lang_keys) / n_langs, 2),
    }

    return result

def build_sample_analysis(samples: list[dict]) -> list[dict]:
    """Build sample_analysis.json entries."""
    out = []
    for s in samples:
        ref = s['reference']
        hyp = s.get('transcript', '') or ''
        rn = norm_standard(ref)
        hn = norm_standard(hyp)
        rnc = norm_numcanon(ref)
        hnc = norm_numcanon(hyp)
        rm = norm_mer(ref)
        hm = norm_mer(hyp)

        flags = []
        if ref == hyp:
            flags.append('exact_match')
        if rn == hn:
            flags.append('exact_match_norm')
        if ref != hyp and rn == hn:
            flags.append('punctuation_only_diff')
        if not hyp.strip():
            flags.append('empty_hypothesis')
        # Numeric mismatch: digits present in either
        if re.search(r'\d', ref) or re.search(r'\d', hyp):
            if rn != hn:
                flags.append('numeric_mismatch')

        out.append({
            'id': s['id'],
            'language': s['language'],
            'reference': ref,
            'hypothesis': hyp,
            'ref_norm': rn,
            'hyp_norm': hn,
            'ref_numcanon': rnc,
            'hyp_numcanon': hnc,
            'detected_language': s.get('detected_lang', ''),
            'flags': flags,
        })
    return out

# ── Model registry ──

MODELS = {
    'elevenlabs_full_6k.jsonl': {
        'model_id': 'elevenlabs-scribe-v2',
        'display_name': 'ElevenLabs Scribe v2',
        'model_type': 'elevenlabs-scribe-v2',
    },
    'elevenlabs_v1_full_6k.jsonl': {
        'model_id': 'elevenlabs-scribe-v1',
        'display_name': 'ElevenLabs Scribe v1',
        'model_type': 'elevenlabs-scribe-v1',
    },
    'gemini_3_flash_strict_full_6k.jsonl': {
        'model_id': 'gemini-3-flash-strict',
        'display_name': 'Gemini 3 Flash (Strict)',
        'model_type': 'gemini-3-flash-preview',
    },
    'gemini_gemini_3_flash_preview_full_6k.jsonl': {
        'model_id': 'gemini-3-flash-preview',
        'display_name': 'Gemini 3 Flash (Preview)',
        'model_type': 'gemini-3-flash-preview',
    },
    'gemini_flash_v2_full_6k.jsonl': {
        'model_id': 'gemini-2.5-flash',
        'display_name': 'Gemini 2.5 Flash',
        'model_type': 'gemini-2.5-flash',
    },
    'gemini_gemini_3_1_pro_preview_full_6k.jsonl': {
        'model_id': 'gemini-3.1-pro',
        'display_name': 'Gemini 3.1 Pro',
        'model_type': 'gemini-3.1-pro-preview',
    },
    'sarvam_full_6k.jsonl': {
        'model_id': 'sarvam-saaras-v3',
        'display_name': 'Sarvam Saaras v3',
        'model_type': 'saaras:v3',
    },
}

OUTPUT_BASE = Path('/home/ubuntu/training/benchmark_outputs')

def process_model(filename: str, info: dict):
    model_id = info['model_id']
    print(f'\n{"="*60}')
    print(f'Processing: {info["display_name"]} ({filename})')
    print(f'{"="*60}')

    with open(filename) as f:
        samples = [json.loads(line) for line in f]

    print(f'  Loaded {len(samples)} samples')

    # Compute metrics
    metrics = compute_metrics_for_samples(samples)

    # Add meta
    total_audio = sum(s.get('duration', 0) for s in samples)
    metrics['__meta__'] = {
        'checkpoint': f'api/{model_id}',
        'checkpoint_name': 'baseline',
        'model_id': model_id,
        'model_type': info['model_type'],
        'dataset': 'BayAreaBoys/indic-asr-benchmark-6k',
        'batch_size': 1,
        'inference_time_sec': 0,
        'total_audio_sec': round(total_audio, 2),
        'rtf': 0,
        'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
        'normalization_version': 'v1',
        'framework': 'api',
    }

    # Print summary
    o = metrics['__overall__']
    print(f'  Overall: wer_raw={o["wer_raw"]:.2f}  wer_norm={o["wer_norm"]:.2f}  wer_numcanon={o["wer_numcanon"]:.2f}  cer_norm={o["cer_norm"]:.2f}')

    # Build sample analysis
    sample_analysis = build_sample_analysis(samples)

    # Write outputs
    out_dir = OUTPUT_BASE / model_id / 'baseline'
    out_dir.mkdir(parents=True, exist_ok=True)

    with open(out_dir / 'metrics.json', 'w') as f:
        json.dump(metrics, f, indent=2, ensure_ascii=False)

    with open(out_dir / 'sample_analysis.json', 'w') as f:
        json.dump(sample_analysis, f, indent=2, ensure_ascii=False)

    print(f'  Written to {out_dir}/')
    return metrics

if __name__ == '__main__':
    results = {}
    for filename, info in MODELS.items():
        fpath = Path(filename)
        if not fpath.exists():
            fpath = Path('/home/ubuntu/training/api_results') / filename
        if not fpath.exists():
            print(f'SKIP: {filename} not found')
            continue
        results[info['model_id']] = process_model(str(fpath), info)

    # Print comparison table
    print(f'\n\n{"="*80}')
    print('COMPARISON TABLE (all models)')
    print(f'{"="*80}')
    print(f'{"Model":<30} {"WER Raw":>8} {"WER Norm":>9} {"WER Num":>8} {"CER Norm":>9}')
    print('-' * 70)
    for mid, m in sorted(results.items(), key=lambda x: x[1]['__overall__']['wer_norm']):
        o = m['__overall__']
        print(f'{mid:<30} {o["wer_raw"]:>7.2f}% {o["wer_norm"]:>8.2f}% {o["wer_numcanon"]:>7.2f}% {o["cer_norm"]:>8.2f}%')


Now update the print summary:


print(f'  Overall: wer_raw={o["wer_raw"]:.2f}  wer_norm={o["wer_norm"]:.2f}  wer_numcanon={o["wer_numcanon"]:.2f}  cer_norm={o["cer_norm"]:.2f}')

print(f'  Overall: wer_raw={o["wer_raw"]:.2f}  wer_norm={o["wer_norm"]:.2f}  mer={o["mer"]:.2f}  cer_norm={o["cer_norm"]:.2f}')


"""
Compute all 4 metric tiers for API baseline models.
Normalization version: v1
"""
import json
import re
import unicodedata
import sys
from collections import defaultdict
from pathlib import Path
from datetime import datetime, timezone

from jiwer import wer as compute_wer, cer as compute_cer

# ── Normalization pipeline ──

# Zero-width characters to strip
ZW_CHARS = re.compile(r'[\u200b-\u200f\u2028-\u202f\ufeff\u00ad]')

# Punctuation sets
# Common punctuation (ASCII + Unicode general)
PUNCT_COMMON = re.compile(r'[!"#$%&\'()*+,\-./:;<=>?@\[\\\]^_`{|}~]')
# Indic punctuation (dandas, etc.) — keep single danda for segmentation? No, remove for norm.
PUNCT_INDIC = re.compile(r'[\u0964\u0965\u0970\u0971]')  # danda, double danda, abbreviation signs
# Extended punctuation
PUNCT_EXTENDED = re.compile(r'[\u2010-\u2027\u2030-\u205e\u2e00-\u2e4f]')
# Quotes normalization
QUOTES = re.compile(r'[\u2018\u2019\u201a\u201b\u201c\u201d\u201e\u201f\u00ab\u00bb]')

# Parenthetical annotations like "(coughing)", "(two seconds pause)"
ANNOTATIONS = re.compile(r'\([^)]*\)')

def norm_raw(text: str) -> str:
    """Step for wer_raw: NFC unicode + trim only."""
    text = unicodedata.normalize('NFC', text)
    text = text.strip()
    return text

def norm_standard(text: str) -> str:
    """Steps for wer_norm: NFKC + strip ZW + whitespace + punct + case fold."""
    # Step 1: NFKC
    text = unicodedata.normalize('NFKC', text)
    # Step 2: Strip zero-width chars
    text = ZW_CHARS.sub('', text)
    # Step 2.5: Remove parenthetical annotations
    text = ANNOTATIONS.sub('', text)
    # Step 3: Normalize whitespace
    text = ' '.join(text.split())
    # Step 4: Standardize quotes/dashes
    text = QUOTES.sub("'", text)
    text = text.replace('\u2014', '-').replace('\u2013', '-').replace('\u2012', '-')
    # Step 5: Remove punctuation
    text = PUNCT_COMMON.sub('', text)
    text = PUNCT_INDIC.sub('', text)
    text = PUNCT_EXTENDED.sub('', text)
    # Step 6: Case fold (affects English only, Indic scripts have no case)
    text = text.lower()
    # Final whitespace cleanup
    text = ' '.join(text.split()).strip()
    return text

def norm_mer(text: str) -> str:
    """Steps for MER: norm_standard + remove ALL spaces."""
    return norm_standard(text).replace(' ', '')

def norm_numcanon(text: str) -> str:
    """Steps for wer_numcanon: norm_standard + number canonicalization."""
    text = norm_standard(text)
    # Remove commas in numbers (e.g., "25,000" -> "25000")
    text = re.sub(r'(\d),(\d)', r'\1\2', text)
    # Normalize Devanagari digits to ASCII
    for offset, base in [(0x0966, '0'), (0x09E6, '0'), (0x0A66, '0'),
                          (0x0AE6, '0'), (0x0B66, '0'), (0x0BE6, '0'),
                          (0x0C66, '0'), (0x0CE6, '0'), (0x0D66, '0')]:
        for i in range(10):
            text = text.replace(chr(offset + i), str(i))
    return text

# ── Metric computation ──

def compute_metrics_for_samples(samples: list[dict]) -> dict:
    """Compute all 4 metric tiers for a list of samples."""
    by_lang = defaultdict(list)
    for s in samples:
        by_lang[s['language']].append(s)

    result = {}
    all_ref_raw, all_hyp_raw = [], []
    all_ref_norm, all_hyp_norm = [], []
    all_ref_nc, all_hyp_nc = [], []
    all_ref_mer, all_hyp_mer = [], []

    for lang in sorted(by_lang.keys()):
        lang_samples = by_lang[lang]
        refs_raw, hyps_raw = [], []
        refs_norm, hyps_norm = [], []
        refs_nc, hyps_nc = [], []
        refs_mer, hyps_mer = [], []
        empty_count = 0

        for s in lang_samples:
            ref = s['reference']
            hyp = s.get('transcript', '') or ''

            if not hyp.strip():
                empty_count += 1

            rr, hr = norm_raw(ref), norm_raw(hyp)
            rn, hn = norm_standard(ref), norm_standard(hyp)
            rnc, hnc = norm_numcanon(ref), norm_numcanon(hyp)
            rm, hm = norm_mer(ref), norm_mer(hyp)

            # Skip empty refs (shouldn't happen but safety)
            if not rr.strip():
                continue

            refs_raw.append(rr); hyps_raw.append(hr if hr.strip() else '<empty>')
            refs_norm.append(rn if rn else '<empty>'); hyps_norm.append(hn if hn else '<empty>')
            refs_nc.append(rnc if rnc else '<empty>'); hyps_nc.append(hnc if hnc else '<empty>')
            refs_mer.append(rm if rm else '<empty>'); hyps_mer.append(hm if hm else '<empty>')

        w_raw = compute_wer(refs_raw, hyps_raw) * 100
        w_norm = compute_wer(refs_norm, hyps_norm) * 100
        w_nc = compute_wer(refs_nc, hyps_nc) * 100
        c_norm = compute_cer(refs_norm, hyps_norm) * 100
        # MER: CER on space-stripped text (each sample is one "word")
        m_er = compute_cer(refs_mer, hyps_mer) * 100

        result[lang] = {
            'n_samples': len(lang_samples),
            'wer_raw': round(w_raw, 2),
            'wer_norm': round(w_norm, 2),
            'wer_numcanon': round(w_nc, 2),
            'mer': round(m_er, 2),
            'cer_norm': round(c_norm, 2),
            'empty_hypotheses': empty_count,
            'normalization_delta': {
                'raw_to_norm': round(w_raw - w_norm, 2),
                'norm_to_numcanon': round(w_norm - w_nc, 2),
                'norm_to_mer': round(w_norm - m_er, 2),
            }
        }

        all_ref_raw.extend(refs_raw); all_hyp_raw.extend(hyps_raw)
        all_ref_norm.extend(refs_norm); all_hyp_norm.extend(hyps_norm)
        all_ref_nc.extend(refs_nc); all_hyp_nc.extend(hyps_nc)
        all_ref_mer.extend(refs_mer); all_hyp_mer.extend(hyps_mer)

    # Overall (micro-average)
    result['__overall__'] = {
        'n_samples': len(samples),
        'wer_raw': round(compute_wer(all_ref_raw, all_hyp_raw) * 100, 2),
        'wer_norm': round(compute_wer(all_ref_norm, all_hyp_norm) * 100, 2),
        'wer_numcanon': round(compute_wer(all_ref_nc, all_hyp_nc) * 100, 2),
        'mer': round(compute_cer(all_ref_mer, all_hyp_mer) * 100, 2),
        'cer_norm': round(compute_cer(all_ref_norm, all_hyp_norm) * 100, 2),
    }

    # Macro average
    lang_keys = [k for k in result if not k.startswith('_')]
    n_langs = len(lang_keys)
    result['__macro_avg__'] = {
        'n_languages': n_langs,
        'wer_raw': round(sum(result[l]['wer_raw'] for l in lang_keys) / n_langs, 2),
        'wer_norm': round(sum(result[l]['wer_norm'] for l in lang_keys) / n_langs, 2),
        'wer_numcanon': round(sum(result[l]['wer_numcanon'] for l in lang_keys) / n_langs, 2),
        'mer': round(sum(result[l]['mer'] for l in lang_keys) / n_langs, 2),
        'cer_norm': round(sum(result[l]['cer_norm'] for l in lang_keys) / n_langs, 2),
    }

    return result

def build_sample_analysis(samples: list[dict]) -> list[dict]:
    """Build sample_analysis.json entries."""
    out = []
    for s in samples:
        ref = s['reference']
        hyp = s.get('transcript', '') or ''
        rn = norm_standard(ref)
        hn = norm_standard(hyp)
        rnc = norm_numcanon(ref)
        hnc = norm_numcanon(hyp)
        rm = norm_mer(ref)
        hm = norm_mer(hyp)

        flags = []
        if ref == hyp:
            flags.append('exact_match')
        if rn == hn:
            flags.append('exact_match_norm')
        if ref != hyp and rn == hn:
            flags.append('punctuation_only_diff')
        if not hyp.strip():
            flags.append('empty_hypothesis')
        # Numeric mismatch: digits present in either
        if re.search(r'\d', ref) or re.search(r'\d', hyp):
            if rn != hn:
                flags.append('numeric_mismatch')

        out.append({
            'id': s['id'],
            'language': s['language'],
            'reference': ref,
            'hypothesis': hyp,
            'ref_norm': rn,
            'hyp_norm': hn,
            'ref_numcanon': rnc,
            'hyp_numcanon': hnc,
            'ref_mer': rm,
            'hyp_mer': hm,
            'detected_language': s.get('detected_lang', ''),
            'flags': flags,
        })
    return out

# ── Model registry ──

MODELS = {
    'elevenlabs_full_6k.jsonl': {
        'model_id': 'elevenlabs-scribe-v2',
        'display_name': 'ElevenLabs Scribe v2',
        'model_type': 'elevenlabs-scribe-v2',
    },
    'elevenlabs_v1_full_6k.jsonl': {
        'model_id': 'elevenlabs-scribe-v1',
        'display_name': 'ElevenLabs Scribe v1',
        'model_type': 'elevenlabs-scribe-v1',
    },
    'gemini_3_flash_strict_full_6k.jsonl': {
        'model_id': 'gemini-3-flash-strict',
        'display_name': 'Gemini 3 Flash (Strict)',
        'model_type': 'gemini-3-flash-preview',
    },
    'gemini_gemini_3_flash_preview_full_6k.jsonl': {
        'model_id': 'gemini-3-flash-preview',
        'display_name': 'Gemini 3 Flash (Preview)',
        'model_type': 'gemini-3-flash-preview',
    },
    'gemini_flash_v2_full_6k.jsonl': {
        'model_id': 'gemini-2.5-flash',
        'display_name': 'Gemini 2.5 Flash',
        'model_type': 'gemini-2.5-flash',
    },
    'gemini_gemini_3_1_pro_preview_full_6k.jsonl': {
        'model_id': 'gemini-3.1-pro',
        'display_name': 'Gemini 3.1 Pro',
        'model_type': 'gemini-3.1-pro-preview',
    },
    'sarvam_full_6k.jsonl': {
        'model_id': 'sarvam-saaras-v3',
        'display_name': 'Sarvam Saaras v3',
        'model_type': 'saaras:v3',
    },
}

OUTPUT_BASE = Path('/home/ubuntu/training/benchmark_outputs')

def process_model(filename: str, info: dict):
    model_id = info['model_id']
    print(f'\n{"="*60}')
    print(f'Processing: {info["display_name"]} ({filename})')
    print(f'{"="*60}')

    with open(filename) as f:
        samples = [json.loads(line) for line in f]

    print(f'  Loaded {len(samples)} samples')

    # Compute metrics
    metrics = compute_metrics_for_samples(samples)

    # Add meta
    total_audio = sum(s.get('duration', 0) for s in samples)
    metrics['__meta__'] = {
        'checkpoint': f'api/{model_id}',
        'checkpoint_name': 'baseline',
        'model_id': model_id,
        'model_type': info['model_type'],
        'dataset': 'BayAreaBoys/indic-asr-benchmark-6k',
        'batch_size': 1,
        'inference_time_sec': 0,
        'total_audio_sec': round(total_audio, 2),
        'rtf': 0,
        'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
        'normalization_version': 'v1',
        'framework': 'api',
    }

    # Print summary
    o = metrics['__overall__']
    print(f'  Overall: wer_raw={o["wer_raw"]:.2f}  wer_norm={o["wer_norm"]:.2f}  wer_numcanon={o["wer_numcanon"]:.2f}  cer_norm={o["cer_norm"]:.2f}')

    # Build sample analysis
    sample_analysis = build_sample_analysis(samples)

    # Write outputs
    out_dir = OUTPUT_BASE / model_id / 'baseline'
    out_dir.mkdir(parents=True, exist_ok=True)

    with open(out_dir / 'metrics.json', 'w') as f:
        json.dump(metrics, f, indent=2, ensure_ascii=False)

    with open(out_dir / 'sample_analysis.json', 'w') as f:
        json.dump(sample_analysis, f, indent=2, ensure_ascii=False)

    print(f'  Written to {out_dir}/')
    return metrics

if __name__ == '__main__':
    results = {}
    for filename, info in MODELS.items():
        fpath = Path(filename)
        if not fpath.exists():
            fpath = Path('/home/ubuntu/training/api_results') / filename
        if not fpath.exists():
            print(f'SKIP: {filename} not found')
            continue
        results[info['model_id']] = process_model(str(fpath), info)

    # Print comparison table
    print(f'\n\n{"="*80}')
    print('COMPARISON TABLE (all models)')
    print(f'{"="*80}')
    print(f'{"Model":<30} {"WER Raw":>8} {"WER Norm":>9} {"WER Num":>8} {"CER Norm":>9}')
    print('-' * 70)
    for mid, m in sorted(results.items(), key=lambda x: x[1]['__overall__']['wer_norm']):
        o = m['__overall__']
        print(f'{mid:<30} {o["wer_raw"]:>7.2f}% {o["wer_norm"]:>8.2f}% {o["wer_numcanon"]:>7.2f}% {o["cer_norm"]:>8.2f}%')

-    print(f'  Overall: wer_raw={o["wer_raw"]:.2f}  wer_norm={o["wer_norm"]:.2f}  wer_numcanon={o["wer_numcanon"]:.2f}  cer_norm={o["cer_norm"]:.2f}')

+    print(f'  Overall: wer_raw={o["wer_raw"]:.2f}  wer_norm={o["wer_norm"]:.2f}  mer={o["mer"]:.2f}  cer_norm={o["cer_norm"]:.2f}')


Now recompute all API baselines + our gemma models (qwen already has MER from the other agent):


Bash

cd /home/ubuntu/training/api_results && python3 compute_metrics.py 2>&1

Recompute all API baselines with MER


============================================================
Processing: ElevenLabs Scribe v2 (elevenlabs_full_6k.jsonl)
============================================================
  Loaded 6000 samples
  Overall: wer_raw=21.94  wer_norm=17.74  mer=5.36  cer_norm=5.28
  Written to /home/ubuntu/training/benchmark_outputs/elevenlabs-scribe-v2/baseline/

============================================================
Processing: ElevenLabs Scribe v1 (elevenlabs_v1_full_6k.jsonl)
============================================================
  Loaded 6000 samples
  Overall: wer_raw=25.82  wer_norm=21.11  mer=6.38  cer_norm=6.29
  Written to /home/ubuntu/training/benchmark_outputs/elevenlabs-scribe-v1/baseline/

============================================================
Processing: Gemini 3 Flash (Strict) (gemini_3_flash_strict_full_6k.jsonl)
============================================================
  Loaded 6000 samples
  Overall: wer_raw=27.62  wer_norm=20.12  mer=5.71  cer_norm=5.79
  Written to /home/ubuntu/training/benchmark_outputs/gemini-3-flash-strict/baseline/

============================================================
Processing: Gemini 3 Flash (Preview) (gemini_gemini_3_flash_preview_full_6k.jsonl)
============================================================
  Loaded 6000 samples
  Overall: wer_raw=71.47  wer_norm=67.68  mer=100.70  cer_norm=95.00
  Written to /home/ubuntu/training/benchmark_outputs/gemini-3-flash-preview/baseline/

============================================================
Processing: Gemini 2.5 Flash (gemini_flash_v2_full_6k.jsonl)
============================================================
  Loaded 6000 samples
  Overall: wer_raw=63.80  wer_norm=57.71  mer=66.67  cer_norm=63.57
  Written to /home/ubuntu/training/benchmark_outputs/gemini-2.5-flash/baseline/

============================================================
Processing: Gemini 3.1 Pro (gemini_gemini_3_1_pro_preview_full_6k.jsonl)
============================================================
  Loaded 6000 samples
  Overall: wer_raw=20.50  wer_norm=18.98  mer=4.80  cer_norm=4.92
  Written to /home/ubuntu/training/benchmark_outputs/gemini-3.1-pro/baseline/

============================================================
Processing: Sarvam Saaras v3 (sarvam_full_6k.jsonl)
============================================================
  Loaded 6000 samples
  Overall: wer_raw=26.36  wer_norm=18.26  mer=5.04  cer_norm=5.01
  Written to /home/ubuntu/training/benchmark_outputs/sarvam-saaras-v3/baseline/

================================================================================
COMPARISON TABLE (all models)
================================================================================
Model                           WER Raw  WER Norm  WER Num  CER Norm
----------------------------------------------------------------------
elevenlabs-scribe-v2             21.94%    17.74%   17.74%     5.28%
sarvam-saaras-v3                 26.36%    18.26%   18.26%     5.01%
gemini-3.1-pro                   20.50%    18.98%   18.98%     4.92%
gemini-3-flash-strict            27.62%    20.12%   20.12%     5.79%
elevenlabs-scribe-v1             25.82%    21.11%   21.11%     6.29%
gemini-2.5-flash                 63.80%    57.71%   57.71%    63.57%
gemini-3-flash-preview           71.47%    67.68%   67.68%    95.00%


Now recompute gemma3n-e2b with MER:


python3 << 'PYEOF'
import json
from pathlib import Path
from datetime import datetime, timezone
from compute_metrics import compute_metrics_for_samples, build_sample_analysis

OUTPUT_BASE = Path('/home/ubuntu/training/benchmark_outputs')

for model_id in ['gemma3n-e2b']:
    model_dir = OUTPUT_BASE / model_id
    for ckpt_dir in sorted(model_dir.iterdir()):
        sa_path = ckpt_dir / 'sample_analysis.json'
        if not sa_path.exists(): continue
        ckpt = ckpt_dir.name
        print(f'Recomputing {model_id}/{ckpt}...')
        raw_samples = json.load(open(sa_path))
        samples = [{'id': s['id'], 'language': s['language'], 'reference': s['reference'],
                     'transcript': s['hypothesis'], 'detected_lang': s.get('detected_language', '')}
                    for s in raw_samples]
        metrics = compute_metrics_for_samples(samples)
        old_meta = json.load(open(ckpt_dir / 'metrics.json')).get('__meta__', {})
        metrics['__meta__'] = {**old_meta, 'model_id': model_id, 'checkpoint_name': ckpt,
                                'normalization_version': 'v1', 'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}
        json.dump(metrics, open(ckpt_dir / 'metrics.json', 'w'), indent=2, ensure_ascii=False)
        new_samples = build_sample_analysis(samples)
        json.dump(new_samples, open(sa_path, 'w'), indent=2, ensure_ascii=False)
        o = metrics['__overall__']
        print(f'  wer_norm={o["wer_norm"]:.2f}  mer={o["mer"]:.2f}  gap={o["wer_norm"]-o["mer"]:.2f}')
PYEOF

Recompute gemma3n-e2b with MER


Recomputing gemma3n-e2b/ckpt-10000...
  wer_norm=36.24  mer=18.09  gap=18.15
Recomputing gemma3n-e2b/ckpt-20000...
  wer_norm=33.95  mer=16.07  gap=17.88


Now fix permissions and update the dashboard types + UI to include MER:


chmod -R o+rx /home/ubuntu/training/benchmark_outputs/

Fix nginx permissions


(Bash completed with no output)


    10→}
    11→
    12→export interface LanguageMetrics {
    13→  n_samples: number;
    14→  wer_raw: number;
    15→  wer_norm: number;
    16→  wer_numcanon: number;
    17→  cer_norm: number;
    18→  empty_hypotheses: number;
    19→  normalization_delta: NormalizationDelta;
    20→}
    21→
    22→export interface AggregateMetrics {
    23→  n_samples: number;
    24→  wer_raw: number;
    25→  wer_norm: number;
    26→  wer_numcanon: number;
    27→  cer_norm: number;
    28→}
    29→
    30→export interface MacroAvgMetrics {
    31→  n_languages: number;
    32→  wer_raw: number;
    33→  wer_norm: number;
    34→  wer_numcanon: number;
    35→  cer_norm: number;
    36→}
    37→
    38→export interface MetaMeta {
    39→  checkpoint: string;

}

export interface LanguageMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  cer_norm: number;
  empty_hypotheses: number;
  normalization_delta: NormalizationDelta;
}

export interface AggregateMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  cer_norm: number;
}

export interface MacroAvgMetrics {
  n_languages: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  cer_norm: number;
}

export interface MetaMeta {
  checkpoint: string;


Edit

export interface NormalizationDelta {
  raw_to_norm: number;       // wer_raw - wer_norm (negative = norm helped)
  norm_to_numcanon: number;  // wer_norm - wer_numcanon
}

export interface LanguageMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  cer_norm: number;
  empty_hypotheses: number;
  normalization_delta: NormalizationDelta;
}

export interface AggregateMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  cer_norm: number;
}

export interface MacroAvgMetrics {
  n_languages: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  cer_norm: number;
}

export interface NormalizationDelta {
  raw_to_norm: number;       // wer_raw - wer_norm (negative = norm helped)
  norm_to_numcanon: number;  // wer_norm - wer_numcanon
  norm_to_mer?: number;      // wer_norm - mer (quantifies spacing error inflation)
}

export interface LanguageMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
  empty_hypotheses: number;
  normalization_delta: NormalizationDelta;
}

export interface AggregateMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
}

export interface MacroAvgMetrics {
  n_languages: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
}


The file /home/ubuntu/training/asr-benchmark-dashboard/src/lib/types.ts has been updated successfully.

export interface NormalizationDelta {
  raw_to_norm: number;       // wer_raw - wer_norm (negative = norm helped)
  norm_to_numcanon: number;  // wer_norm - wer_numcanon
  norm_to_mer?: number;      // wer_norm - mer (quantifies spacing error inflation)
}

export interface LanguageMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
  empty_hypotheses: number;
  normalization_delta: NormalizationDelta;
}

export interface AggregateMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
}

export interface MacroAvgMetrics {
  n_languages: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
}

// === Metric tiers ===
// wer_raw:      minimal normalization, strict fidelity
// wer_norm:     PRIMARY metric — NFKC + whitespace + punctuation + case fold
// wer_numcanon: wer_norm + number canonicalization
// cer_norm:     character-level normalized — critical for Indic scripts

export interface NormalizationDelta {
  raw_to_norm: number;       // wer_raw - wer_norm (negative = norm helped)
  norm_to_numcanon: number;  // wer_norm - wer_numcanon
}

export interface LanguageMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  cer_norm: number;
  empty_hypotheses: number;
  normalization_delta: NormalizationDelta;
}

export interface AggregateMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  cer_norm: number;
}

export interface MacroAvgMetrics {
  n_languages: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  cer_norm: number;
}

export interface MetaMeta {
  checkpoint: string;
  checkpoint_name: string;
  model_id: string;
  model_type?: string;
  dataset: string;
  batch_size?: number;
  force_language?: boolean;
  inference_time_sec: number;
  total_audio_sec: number;
  rtf: number;
  timestamp?: string;
  gpu?: string;
  framework?: string;
  normalization_version?: string;
  jiwer_version?: string;
}

export type MetricsFile = {
  [language: string]: LanguageMetrics | AggregateMetrics | MacroAvgMetrics | MetaMeta;
} & {
  __overall__: AggregateMetrics;
  __macro_avg__: MacroAvgMetrics;
  __meta__: MetaMeta;
};

export interface SampleAnalysis {
  id: string;
  language: string;
  reference: string;
  hypothesis: string;
  ref_norm?: string;
  hyp_norm?: string;
  ref_numcanon?: string;
  hyp_numcanon?: string;
  detected_language?: string;
  wer_raw?: number;
  wer_norm?: number;
  flags?: string[];
}

export interface ErrorBuckets {
  numeric_mismatch_count: number;
  punctuation_only_count: number;
  spacing_tokenization_count: number;
  entity_mismatch_count: number;
  script_confusion_count: number;
  empty_hypothesis_count: number;
}

export interface SubstitutionEntry {
  ref: string;
  hyp: string;
  count: number;
}

export interface WordCountEntry {
  word: string;
  count: number;
}

export interface LanguageErrorAnalysis {
  top_substitutions: SubstitutionEntry[];
  top_insertions: WordCountEntry[];
  top_deletions: WordCountEntry[];
  error_buckets: ErrorBuckets;
  examples: {
    worst_samples: string[];
    best_samples: string[];
    numeric_mismatch_samples: string[];
    entity_mismatch_samples: string[];
  };
}

export interface ErrorAnalysisSummary {
  model_diagnosis: "recognition-limited" | "formatting-limited" | "numeric-limited" | "mixed";
  primary_error_source: string;
  numeric_verbalization_impact: string;
  formatting_impact: string;
  worst_languages: string[];
  best_languages: string[];
}

export type ErrorAnalysisFile = {
  [language: string]: LanguageErrorAnalysis | ErrorAnalysisSummary;
} & {
  __summary__: ErrorAnalysisSummary;
};

export interface ModelInfo {
  id: string;
  displayName: string;
  checkpoints: string[];
}

export interface CheckpointComparison {
  language: string;
  [key: string]: number | string;
}

// === Constants ===

export const MODEL_REGISTRY: Record<string, string> = {
  "qwen3-asr": "Qwen3-ASR-1.7B",
  "gemma3n-e2b": "Gemma-3n-E2B-ASR",
  "parakeet-1.1b": "Parakeet-1.1B-Language-Guided",
  "elevenlabs-scribe-v2": "ElevenLabs Scribe v2",
  "elevenlabs-scribe-v1": "ElevenLabs Scribe v1",
  "gemini-3-flash-strict": "Gemini 3 Flash (Strict)",
  "gemini-3.1-pro": "Gemini 3.1 Pro",
  "sarvam-saaras-v3": "Sarvam Saaras v3",
};

// Models grouped by category
export const OUR_MODELS = ["qwen3-asr", "gemma3n-e2b", "parakeet-1.1b"] as const;
export const BASELINE_MODELS = [
  "elevenlabs-scribe-v2", "elevenlabs-scribe-v1",
  "gemini-3-flash-strict", "gemini-3.1-pro",
  "sarvam-saaras-v3",
] as const;

export const LANGUAGES = [
  "assamese", "bengali", "english", "gujarati", "hindi", "kannada",
  "malayalam", "marathi", "odia", "punjabi", "tamil", "telugu",
] as const;

export const LANGUAGE_CODES: Record<string, string> = {
  assamese: "as", bengali: "bn", english: "en", gujarati: "gu",
  hindi: "hi", kannada: "kn", malayalam: "ml", marathi: "mr",
  odia: "or", punjabi: "pa", tamil: "ta", telugu: "te",
};

export const METRIC_TIERS = ["wer_raw", "wer_norm", "wer_numcanon", "cer_norm"] as const;
export type MetricTier = typeof METRIC_TIERS[number];

export const METRIC_LABELS: Record<MetricTier, string> = {
  wer_raw: "WER Raw",
  wer_norm: "WER Norm",
  wer_numcanon: "WER NumCanon",
  cer_norm: "CER Norm",
};

export const METRIC_DESCRIPTIONS: Record<MetricTier, string> = {
  wer_raw: "Strict transcript fidelity",
  wer_norm: "Primary ASR metric",
  wer_numcanon: "Numeric robustness",
  cer_norm: "Script-sensitive (Indic)",
};

export const CHART_COLORS = [
  "#6366f1", "#8b5cf6", "#a78bfa", "#c084fc",
  "#e879f9", "#f472b6", "#fb7185", "#f87171",
  "#fb923c", "#fbbf24", "#a3e635", "#34d399",
];

export const MODEL_COLORS: Record<string, string> = {
  "qwen3-asr": "#6366f1",
  "gemma3n-e2b": "#8b5cf6",
  "parakeet-1.1b": "#34d399",
  "elevenlabs-scribe-v2": "#f472b6",
  "elevenlabs-scribe-v1": "#fb7185",
  "gemini-3-flash-strict": "#fbbf24",
  "gemini-3.1-pro": "#a3e635",
  "sarvam-saaras-v3": "#22d3ee",
};

// === Backward compatibility ===
// Converts old-schema metrics into new-schema shape so dashboard works
// with both old data (pre-migration) and new data.

export interface OldLanguageMetrics {
  n_samples: number;
  wer: number;
  cer: number;
  wer_normalized: number;
  cer_normalized: number;
  empty_hypotheses: number;
}

export function isOldSchema(obj: Record<string, unknown>): boolean {
  return "wer" in obj && !("wer_raw" in obj);
}

export function migrateLanguageMetrics(old: OldLanguageMetrics): LanguageMetrics {
  return {
    n_samples: old.n_samples,
    wer_raw: old.wer,
    wer_norm: old.wer_normalized,
    wer_numcanon: old.wer_normalized, // no numcanon data in old schema, use norm as fallback
    cer_norm: old.cer_normalized,
    empty_hypotheses: old.empty_hypotheses,
    normalization_delta: {
      raw_to_norm: old.wer - old.wer_normalized,
      norm_to_numcanon: 0,
    },
  };
}

+  norm_to_mer?: number;      // wer_norm - mer (quantifies spacing error inflation)

wer_raw: number;
wer_norm: number;
wer_numcanon: number;
+  mer: number;
cer_norm: number;
}


Now add MER to the metric tiers constants:


export const METRIC_TIERS = ["wer_raw", "wer_norm", "wer_numcanon", "cer_norm"] as const;
export type MetricTier = typeof METRIC_TIERS[number];

export const METRIC_LABELS: Record<MetricTier, string> = {
  wer_raw: "WER Raw",
  wer_norm: "WER Norm",
  wer_numcanon: "WER NumCanon",
  cer_norm: "CER Norm",
};

export const METRIC_DESCRIPTIONS: Record<MetricTier, string> = {
  wer_raw: "Strict transcript fidelity",
  wer_norm: "Primary ASR metric",
  wer_numcanon: "Numeric robustness",
  cer_norm: "Script-sensitive (Indic)",
};

export const METRIC_TIERS = ["wer_raw", "wer_norm", "wer_numcanon", "mer", "cer_norm"] as const;
export type MetricTier = typeof METRIC_TIERS[number];

export const METRIC_LABELS: Record<MetricTier, string> = {
  wer_raw: "WER Raw",
  wer_norm: "WER Norm",
  wer_numcanon: "WER NumCanon",
  mer: "MER",
  cer_norm: "CER Norm",
};

export const METRIC_DESCRIPTIONS: Record<MetricTier, string> = {
  wer_raw: "Strict transcript fidelity",
  wer_norm: "Primary ASR metric",
  wer_numcanon: "Numeric robustness",
  mer: "Meaningful Error Rate (space-normalized)",
  cer_norm: "Script-sensitive (Indic)",
};


export const METRIC_TIERS = ["wer_raw", "wer_norm", "wer_numcanon", "mer", "cer_norm"] as const;
export type MetricTier = typeof METRIC_TIERS[number];

export const METRIC_LABELS: Record<MetricTier, string> = {
  wer_raw: "WER Raw",
  wer_norm: "WER Norm",
  wer_numcanon: "WER NumCanon",
  mer: "MER",
  cer_norm: "CER Norm",
};

export const METRIC_DESCRIPTIONS: Record<MetricTier, string> = {
  wer_raw: "Strict transcript fidelity",
  wer_norm: "Primary ASR metric",
  wer_numcanon: "Numeric robustness",
  mer: "Meaningful Error Rate (space-normalized)",
  cer_norm: "Script-sensitive (Indic)",
};

// === Metric tiers ===
// wer_raw:      minimal normalization, strict fidelity
// wer_norm:     PRIMARY metric — NFKC + whitespace + punctuation + case fold
// wer_numcanon: wer_norm + number canonicalization
// cer_norm:     character-level normalized — critical for Indic scripts

export interface NormalizationDelta {
  raw_to_norm: number;       // wer_raw - wer_norm (negative = norm helped)
  norm_to_numcanon: number;  // wer_norm - wer_numcanon
  norm_to_mer?: number;      // wer_norm - mer (quantifies spacing error inflation)
}

export interface LanguageMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
  empty_hypotheses: number;
  normalization_delta: NormalizationDelta;
}

export interface AggregateMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
}

export interface MacroAvgMetrics {
  n_languages: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
}

export interface MetaMeta {
  checkpoint: string;
  checkpoint_name: string;
  model_id: string;
  model_type?: string;
  dataset: string;
  batch_size?: number;
  force_language?: boolean;
  inference_time_sec: number;
  total_audio_sec: number;
  rtf: number;
  timestamp?: string;
  gpu?: string;
  framework?: string;
  normalization_version?: string;
  jiwer_version?: string;
}

export type MetricsFile = {
  [language: string]: LanguageMetrics | AggregateMetrics | MacroAvgMetrics | MetaMeta;
} & {
  __overall__: AggregateMetrics;
  __macro_avg__: MacroAvgMetrics;
  __meta__: MetaMeta;
};

export interface SampleAnalysis {
  id: string;
  language: string;
  reference: string;
  hypothesis: string;
  ref_norm?: string;
  hyp_norm?: string;
  ref_numcanon?: string;
  hyp_numcanon?: string;
  detected_language?: string;
  wer_raw?: number;
  wer_norm?: number;
  flags?: string[];
}

export interface ErrorBuckets {
  numeric_mismatch_count: number;
  punctuation_only_count: number;
  spacing_tokenization_count: number;
  entity_mismatch_count: number;
  script_confusion_count: number;
  empty_hypothesis_count: number;
}

export interface SubstitutionEntry {
  ref: string;
  hyp: string;
  count: number;
}

export interface WordCountEntry {
  word: string;
  count: number;
}

export interface LanguageErrorAnalysis {
  top_substitutions: SubstitutionEntry[];
  top_insertions: WordCountEntry[];
  top_deletions: WordCountEntry[];
  error_buckets: ErrorBuckets;
  examples: {
    worst_samples: string[];
    best_samples: string[];
    numeric_mismatch_samples: string[];
    entity_mismatch_samples: string[];
  };
}

export interface ErrorAnalysisSummary {
  model_diagnosis: "recognition-limited" | "formatting-limited" | "numeric-limited" | "mixed";
  primary_error_source: string;
  numeric_verbalization_impact: string;
  formatting_impact: string;
  worst_languages: string[];
  best_languages: string[];
}

export type ErrorAnalysisFile = {
  [language: string]: LanguageErrorAnalysis | ErrorAnalysisSummary;
} & {
  __summary__: ErrorAnalysisSummary;
};

export interface ModelInfo {
  id: string;
  displayName: string;
  checkpoints: string[];
}

export interface CheckpointComparison {
  language: string;
  [key: string]: number | string;
}

// === Constants ===

export const MODEL_REGISTRY: Record<string, string> = {
  "qwen3-asr": "Qwen3-ASR-1.7B",
  "gemma3n-e2b": "Gemma-3n-E2B-ASR",
  "parakeet-1.1b": "Parakeet-1.1B-Language-Guided",
  "elevenlabs-scribe-v2": "ElevenLabs Scribe v2",
  "elevenlabs-scribe-v1": "ElevenLabs Scribe v1",
  "gemini-3-flash-strict": "Gemini 3 Flash (Strict)",
  "gemini-3.1-pro": "Gemini 3.1 Pro",
  "sarvam-saaras-v3": "Sarvam Saaras v3",
};

// Models grouped by category
export const OUR_MODELS = ["qwen3-asr", "gemma3n-e2b", "parakeet-1.1b"] as const;
export const BASELINE_MODELS = [
  "elevenlabs-scribe-v2", "elevenlabs-scribe-v1",
  "gemini-3-flash-strict", "gemini-3.1-pro",
  "sarvam-saaras-v3",
] as const;

export const LANGUAGES = [
  "assamese", "bengali", "english", "gujarati", "hindi", "kannada",
  "malayalam", "marathi", "odia", "punjabi", "tamil", "telugu",
] as const;

export const LANGUAGE_CODES: Record<string, string> = {
  assamese: "as", bengali: "bn", english: "en", gujarati: "gu",
  hindi: "hi", kannada: "kn", malayalam: "ml", marathi: "mr",
  odia: "or", punjabi: "pa", tamil: "ta", telugu: "te",
};

export const METRIC_TIERS = ["wer_raw", "wer_norm", "wer_numcanon", "cer_norm"] as const;
export type MetricTier = typeof METRIC_TIERS[number];

export const METRIC_LABELS: Record<MetricTier, string> = {
  wer_raw: "WER Raw",
  wer_norm: "WER Norm",
  wer_numcanon: "WER NumCanon",
  cer_norm: "CER Norm",
};

export const METRIC_DESCRIPTIONS: Record<MetricTier, string> = {
  wer_raw: "Strict transcript fidelity",
  wer_norm: "Primary ASR metric",
  wer_numcanon: "Numeric robustness",
  cer_norm: "Script-sensitive (Indic)",
};

export const CHART_COLORS = [
  "#6366f1", "#8b5cf6", "#a78bfa", "#c084fc",
  "#e879f9", "#f472b6", "#fb7185", "#f87171",
  "#fb923c", "#fbbf24", "#a3e635", "#34d399",
];

export const MODEL_COLORS: Record<string, string> = {
  "qwen3-asr": "#6366f1",
  "gemma3n-e2b": "#8b5cf6",
  "parakeet-1.1b": "#34d399",
  "elevenlabs-scribe-v2": "#f472b6",
  "elevenlabs-scribe-v1": "#fb7185",
  "gemini-3-flash-strict": "#fbbf24",
  "gemini-3.1-pro": "#a3e635",
  "sarvam-saaras-v3": "#22d3ee",
};

// === Backward compatibility ===
// Converts old-schema metrics into new-schema shape so dashboard works
// with both old data (pre-migration) and new data.

export interface OldLanguageMetrics {
  n_samples: number;
  wer: number;
  cer: number;
  wer_normalized: number;
  cer_normalized: number;
  empty_hypotheses: number;
}

export function isOldSchema(obj: Record<string, unknown>): boolean {
  return "wer" in obj && !("wer_raw" in obj);
}

export function migrateLanguageMetrics(old: OldLanguageMetrics): LanguageMetrics {
  return {
    n_samples: old.n_samples,
    wer_raw: old.wer,
    wer_norm: old.wer_normalized,
    wer_numcanon: old.wer_normalized, // no numcanon data in old schema, use norm as fallback
    cer_norm: old.cer_normalized,
    empty_hypotheses: old.empty_hypotheses,
    normalization_delta: {
      raw_to_norm: old.wer - old.wer_normalized,
      norm_to_numcanon: 0,
    },
  };
}

-export const METRIC_TIERS = ["wer_raw", "wer_norm", "wer_numcanon", "cer_norm"] as const;

+export const METRIC_TIERS = ["wer_raw", "wer_norm", "wer_numcanon", "mer", "cer_norm"] as const;

+  mer: "Meaningful Error Rate (space-normalized)",


Now update the backward compatibility migration to include MER:


export function migrateLanguageMetrics(old: OldLanguageMetrics): LanguageMetrics {
  return {
    n_samples: old.n_samples,
    wer_raw: old.wer,
    wer_norm: old.wer_normalized,
    wer_numcanon: old.wer_normalized, // no numcanon data in old schema, use norm as fallback
    cer_norm: old.cer_normalized,
    empty_hypotheses: old.empty_hypotheses,
    normalization_delta: {
      raw_to_norm: old.wer - old.wer_normalized,
      norm_to_numcanon: 0,
    },
  };
}

export function migrateLanguageMetrics(old: OldLanguageMetrics): LanguageMetrics {
  return {
    n_samples: old.n_samples,
    wer_raw: old.wer,
    wer_norm: old.wer_normalized,
    wer_numcanon: old.wer_normalized,
    mer: old.cer_normalized, // fallback: use CER as rough MER proxy
    cer_norm: old.cer_normalized,
    empty_hypotheses: old.empty_hypotheses,
    normalization_delta: {
      raw_to_norm: old.wer - old.wer_normalized,
      norm_to_numcanon: 0,
      norm_to_mer: 0,
    },
  };
}


export function migrateLanguageMetrics(old: OldLanguageMetrics): LanguageMetrics {
  return {
    n_samples: old.n_samples,
    wer_raw: old.wer,
    wer_norm: old.wer_normalized,
    wer_numcanon: old.wer_normalized,
    mer: old.cer_normalized, // fallback: use CER as rough MER proxy
    cer_norm: old.cer_normalized,
    empty_hypotheses: old.empty_hypotheses,
    normalization_delta: {
      raw_to_norm: old.wer - old.wer_normalized,
      norm_to_numcanon: 0,
      norm_to_mer: 0,
    },
  };
}

// === Metric tiers ===
// wer_raw:      minimal normalization, strict fidelity
// wer_norm:     PRIMARY metric — NFKC + whitespace + punctuation + case fold
// wer_numcanon: wer_norm + number canonicalization
// cer_norm:     character-level normalized — critical for Indic scripts

export interface NormalizationDelta {
  raw_to_norm: number;       // wer_raw - wer_norm (negative = norm helped)
  norm_to_numcanon: number;  // wer_norm - wer_numcanon
  norm_to_mer?: number;      // wer_norm - mer (quantifies spacing error inflation)
}

export interface LanguageMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
  empty_hypotheses: number;
  normalization_delta: NormalizationDelta;
}

export interface AggregateMetrics {
  n_samples: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
}

export interface MacroAvgMetrics {
  n_languages: number;
  wer_raw: number;
  wer_norm: number;
  wer_numcanon: number;
  mer: number;
  cer_norm: number;
}

export interface MetaMeta {
  checkpoint: string;
  checkpoint_name: string;
  model_id: string;
  model_type?: string;
  dataset: string;
  batch_size?: number;
  force_language?: boolean;
  inference_time_sec: number;
  total_audio_sec: number;
  rtf: number;
  timestamp?: string;
  gpu?: string;
  framework?: string;
  normalization_version?: string;
  jiwer_version?: string;
}

export type MetricsFile = {
  [language: string]: LanguageMetrics | AggregateMetrics | MacroAvgMetrics | MetaMeta;
} & {
  __overall__: AggregateMetrics;
  __macro_avg__: MacroAvgMetrics;
  __meta__: MetaMeta;
};

export interface SampleAnalysis {
  id: string;
  language: string;
  reference: string;
  hypothesis: string;
  ref_norm?: string;
  hyp_norm?: string;
  ref_numcanon?: string;
  hyp_numcanon?: string;
  detected_language?: string;
  wer_raw?: number;
  wer_norm?: number;
  flags?: string[];
}

export interface ErrorBuckets {
  numeric_mismatch_count: number;
  punctuation_only_count: number;
  spacing_tokenization_count: number;
  entity_mismatch_count: number;
  script_confusion_count: number;
  empty_hypothesis_count: number;
}

export interface SubstitutionEntry {
  ref: string;
  hyp: string;
  count: number;
}

export interface WordCountEntry {
  word: string;
  count: number;
}

export interface LanguageErrorAnalysis {
  top_substitutions: SubstitutionEntry[];
  top_insertions: WordCountEntry[];
  top_deletions: WordCountEntry[];
  error_buckets: ErrorBuckets;
  examples: {
    worst_samples: string[];
    best_samples: string[];
    numeric_mismatch_samples: string[];
    entity_mismatch_samples: string[];
  };
}

export interface ErrorAnalysisSummary {
  model_diagnosis: "recognition-limited" | "formatting-limited" | "numeric-limited" | "mixed";
  primary_error_source: string;
  numeric_verbalization_impact: string;
  formatting_impact: string;
  worst_languages: string[];
  best_languages: string[];
}

export type ErrorAnalysisFile = {
  [language: string]: LanguageErrorAnalysis | ErrorAnalysisSummary;
} & {
  __summary__: ErrorAnalysisSummary;
};

export interface ModelInfo {
  id: string;
  displayName: string;
  checkpoints: string[];
}

export interface CheckpointComparison {
  language: string;
  [key: string]: number | string;
}

// === Constants ===

export const MODEL_REGISTRY: Record<string, string> = {
  "qwen3-asr": "Qwen3-ASR-1.7B",
  "gemma3n-e2b": "Gemma-3n-E2B-ASR",
  "parakeet-1.1b": "Parakeet-1.1B-Language-Guided",
  "elevenlabs-scribe-v2": "ElevenLabs Scribe v2",
  "elevenlabs-scribe-v1": "ElevenLabs Scribe v1",
  "gemini-3-flash-strict": "Gemini 3 Flash (Strict)",
  "gemini-3.1-pro": "Gemini 3.1 Pro",
  "sarvam-saaras-v3": "Sarvam Saaras v3",
};

// Models grouped by category
export const OUR_MODELS = ["qwen3-asr", "gemma3n-e2b", "parakeet-1.1b"] as const;
export const BASELINE_MODELS = [
  "elevenlabs-scribe-v2", "elevenlabs-scribe-v1",
  "gemini-3-flash-strict", "gemini-3.1-pro",
  "sarvam-saaras-v3",
] as const;

export const LANGUAGES = [
  "assamese", "bengali", "english", "gujarati", "hindi", "kannada",
  "malayalam", "marathi", "odia", "punjabi", "tamil", "telugu",
] as const;

export const LANGUAGE_CODES: Record<string, string> = {
  assamese: "as", bengali: "bn", english: "en", gujarati: "gu",
  hindi: "hi", kannada: "kn", malayalam: "ml", marathi: "mr",
  odia: "or", punjabi: "pa", tamil: "ta", telugu: "te",
};

export const METRIC_TIERS = ["wer_raw", "wer_norm", "wer_numcanon", "mer", "cer_norm"] as const;
export type MetricTier = typeof METRIC_TIERS[number];

export const METRIC_LABELS: Record<MetricTier, string> = {
  wer_raw: "WER Raw",
  wer_norm: "WER Norm",
  wer_numcanon: "WER NumCanon",
  mer: "MER",
  cer_norm: "CER Norm",
};

export const METRIC_DESCRIPTIONS: Record<MetricTier, string> = {
  wer_raw: "Strict transcript fidelity",
  wer_norm: "Primary ASR metric",
  wer_numcanon: "Numeric robustness",
  mer: "Meaningful Error Rate (space-normalized)",
  cer_norm: "Script-sensitive (Indic)",
};

export const CHART_COLORS = [
  "#6366f1", "#8b5cf6", "#a78bfa", "#c084fc",
  "#e879f9", "#f472b6", "#fb7185", "#f87171",
  "#fb923c", "#fbbf24", "#a3e635", "#34d399",
];

export const MODEL_COLORS: Record<string, string> = {
  "qwen3-asr": "#6366f1",
  "gemma3n-e2b": "#8b5cf6",
  "parakeet-1.1b": "#34d399",
  "elevenlabs-scribe-v2": "#f472b6",
  "elevenlabs-scribe-v1": "#fb7185",
  "gemini-3-flash-strict": "#fbbf24",
  "gemini-3.1-pro": "#a3e635",
  "sarvam-saaras-v3": "#22d3ee",
};

// === Backward compatibility ===
// Converts old-schema metrics into new-schema shape so dashboard works
// with both old data (pre-migration) and new data.

export interface OldLanguageMetrics {
  n_samples: number;
  wer: number;
  cer: number;
  wer_normalized: number;
  cer_normalized: number;
  empty_hypotheses: number;
}

export function isOldSchema(obj: Record<string, unknown>): boolean {
  return "wer" in obj && !("wer_raw" in obj);
}

export function migrateLanguageMetrics(old: OldLanguageMetrics): LanguageMetrics {
  return {
    n_samples: old.n_samples,
    wer_raw: old.wer,
    wer_norm: old.wer_normalized,
    wer_numcanon: old.wer_normalized, // no numcanon data in old schema, use norm as fallback
    cer_norm: old.cer_normalized,
    empty_hypotheses: old.empty_hypotheses,
    normalization_delta: {
      raw_to_norm: old.wer - old.wer_normalized,
      norm_to_numcanon: 0,
    },
  };
}

-    wer_numcanon: old.wer_normalized, // no numcanon data in old schema, use norm as fallback

+    mer: old.cer_normalized, // fallback: use CER as rough MER proxy


Update the data.ts migration for aggregates:


    30→  if (!firstLang || !isOldSchema(raw[firstLang] as Record<string, unknown>)) {
    31→    return raw as unknown as MetricsFile;
    32→  }
    33→
    34→  const migrated: Record<string, unknown> = {};
    35→
    36→  for (const lang of LANGUAGES) {
    37→    if (raw[lang]) {
    38→      migrated[lang] = migrateLanguageMetrics(raw[lang] as OldLanguageMetrics);
    39→    }
    40→  }
    41→
    42→  // Migrate __overall__
    43→  const oldOverall = raw.__overall__ as { n_samples: number; wer: number; cer: number; wer_normalized: number; cer_normalized: number } | undefined;
    44→  if (oldOverall) {
    45→    migrated.__overall__ = {
    46→      n_samples: oldOverall.n_samples,
    47→      wer_raw: oldOverall.wer,
    48→      wer_norm: oldOverall.wer_normalized,
    49→      wer_numcanon: oldOverall.wer_normalized,
    50→      cer_norm: oldOverall.cer_normalized,
    51→    } as AggregateMetrics;
    52→  }
    53→
    54→  // Migrate __macro_avg__
    55→  const oldMacro = raw.__macro_avg__ as { n_languages: number; wer: number; cer: number; wer_normalized: number; cer_normalized: number } | undefined;
    56→  if (oldMacro) {
    57→    migrated.__macro_avg__ = {
    58→      n_languages: oldMacro.n_languages,
    59→      wer_raw: oldMacro.wer,
    60→      wer_norm: oldMacro.wer_normalized,
    61→      wer_numcanon: oldMacro.wer_normalized,
    62→      cer_norm: oldMacro.cer_normalized,
    63→    } as MacroAvgMetrics;
    64→  }

  if (!firstLang || !isOldSchema(raw[firstLang] as Record<string, unknown>)) {
    return raw as unknown as MetricsFile;
  }

  const migrated: Record<string, unknown> = {};

  for (const lang of LANGUAGES) {
    if (raw[lang]) {
      migrated[lang] = migrateLanguageMetrics(raw[lang] as OldLanguageMetrics);
    }
  }

  // Migrate __overall__
  const oldOverall = raw.__overall__ as { n_samples: number; wer: number; cer: number; wer_normalized: number; cer_normalized: number } | undefined;
  if (oldOverall) {
    migrated.__overall__ = {
      n_samples: oldOverall.n_samples,
      wer_raw: oldOverall.wer,
      wer_norm: oldOverall.wer_normalized,
      wer_numcanon: oldOverall.wer_normalized,
      cer_norm: oldOverall.cer_normalized,
    } as AggregateMetrics;
  }

  // Migrate __macro_avg__
  const oldMacro = raw.__macro_avg__ as { n_languages: number; wer: number; cer: number; wer_normalized: number; cer_normalized: number } | undefined;
  if (oldMacro) {
    migrated.__macro_avg__ = {
      n_languages: oldMacro.n_languages,
      wer_raw: oldMacro.wer,
      wer_norm: oldMacro.wer_normalized,
      wer_numcanon: oldMacro.wer_normalized,
      cer_norm: oldMacro.cer_normalized,
    } as MacroAvgMetrics;
  }


      wer_numcanon: oldOverall.wer_normalized,
      cer_norm: oldOverall.cer_normalized,
    } as AggregateMetrics;
  }

  // Migrate __macro_avg__
  const oldMacro = raw.__macro_avg__ as { n_languages: number; wer: number; cer: number; wer_normalized: number; cer_normalized: number } | undefined;
  if (oldMacro) {
    migrated.__macro_avg__ = {
      n_languages: oldMacro.n_languages,
      wer_raw: oldMacro.wer,
      wer_norm: oldMacro.wer_normalized,
      wer_numcanon: oldMacro.wer_normalized,
      cer_norm: oldMacro.cer_normalized,
    } as MacroAvgMetrics;

      wer_numcanon: oldOverall.wer_normalized,
      mer: oldOverall.cer_normalized,
      cer_norm: oldOverall.cer_normalized,
    } as AggregateMetrics;
  }

  // Migrate __macro_avg__
  const oldMacro = raw.__macro_avg__ as { n_languages: number; wer: number; cer: number; wer_normalized: number; cer_normalized: number } | undefined;
  if (oldMacro) {
    migrated.__macro_avg__ = {
      n_languages: oldMacro.n_languages,
      wer_raw: oldMacro.wer,
      wer_norm: oldMacro.wer_normalized,
      wer_numcanon: oldMacro.wer_normalized,
      mer: oldMacro.cer_normalized,
      cer_norm: oldMacro.cer_normalized,
    } as MacroAvgMetrics;


The file /home/ubuntu/training/asr-benchmark-dashboard/src/lib/data.ts has been updated successfully.

      wer_numcanon: oldOverall.wer_normalized,
      mer: oldOverall.cer_normalized,
      cer_norm: oldOverall.cer_normalized,
    } as AggregateMetrics;
  }

  // Migrate __macro_avg__
  const oldMacro = raw.__macro_avg__ as { n_languages: number; wer: number; cer: number; wer_normalized: number; cer_normalized: number } | undefined;
  if (oldMacro) {
    migrated.__macro_avg__ = {
      n_languages: oldMacro.n_languages,
      wer_raw: oldMacro.wer,
      wer_norm: oldMacro.wer_normalized,
      wer_numcanon: oldMacro.wer_normalized,
      mer: oldMacro.cer_normalized,
      cer_norm: oldMacro.cer_normalized,
    } as MacroAvgMetrics;

import {
  MetricsFile, SampleAnalysis, ErrorAnalysisFile,
  ModelInfo, MODEL_REGISTRY, LANGUAGES,
  isOldSchema, migrateLanguageMetrics, OldLanguageMetrics,
  AggregateMetrics, MacroAvgMetrics,
} from "./types";

const BENCHMARK_API = process.env.BENCHMARK_API_URL || "http://localhost:8787";

interface NginxDirEntry {
  name: string;
  type: "directory" | "file";
  mtime: string;
}

async function fetchJson<T>(url: string): Promise<T | null> {
  try {
    const res = await fetch(url, { cache: "no-store" });
    if (!res.ok) return null;
    return (await res.json()) as T;
  } catch {
    return null;
  }
}

// Migrate old-schema metrics.json into new shape
function migrateMetricsFile(raw: Record<string, unknown>): MetricsFile {
  // Check if this is old schema by inspecting a language key
  const firstLang = LANGUAGES.find((l) => l in raw);
  if (!firstLang || !isOldSchema(raw[firstLang] as Record<string, unknown>)) {
    return raw as unknown as MetricsFile;
  }

  const migrated: Record<string, unknown> = {};

  for (const lang of LANGUAGES) {
    if (raw[lang]) {
      migrated[lang] = migrateLanguageMetrics(raw[lang] as OldLanguageMetrics);
    }
  }

  // Migrate __overall__
  const oldOverall = raw.__overall__ as { n_samples: number; wer: number; cer: number; wer_normalized: number; cer_normalized: number } | undefined;
  if (oldOverall) {
    migrated.__overall__ = {
      n_samples: oldOverall.n_samples,
      wer_raw: oldOverall.wer,
      wer_norm: oldOverall.wer_normalized,
      wer_numcanon: oldOverall.wer_normalized,
      cer_norm: oldOverall.cer_normalized,
    } as AggregateMetrics;
  }

  // Migrate __macro_avg__
  const oldMacro = raw.__macro_avg__ as { n_languages: number; wer: number; cer: number; wer_normalized: number; cer_normalized: number } | undefined;
  if (oldMacro) {
    migrated.__macro_avg__ = {
      n_languages: oldMacro.n_languages,
      wer_raw: oldMacro.wer,
      wer_norm: oldMacro.wer_normalized,
      wer_numcanon: oldMacro.wer_normalized,
      cer_norm: oldMacro.cer_normalized,
    } as MacroAvgMetrics;
  }

  migrated.__meta__ = raw.__meta__;

  return migrated as unknown as MetricsFile;
}

// Migrate old sample_analysis entries
function migrateSamples(raw: Record<string, unknown>[]): SampleAnalysis[] {
  return raw.map((s) => {
    const sample = s as Record<string, string>;
    return {
      id: sample.id,
      language: sample.language,
      reference: sample.reference,
      hypothesis: sample.hypothesis,
      ref_norm: sample.ref_norm || sample.ref_normalized,
      hyp_norm: sample.hyp_norm || sample.hyp_normalized,
      detected_language: sample.detected_language,
    } as SampleAnalysis;
  });
}

export async function getModels(): Promise<ModelInfo[]> {
  const models: ModelInfo[] = [];

  for (const [modelId, displayName] of Object.entries(MODEL_REGISTRY)) {
    const entries = await fetchJson<NginxDirEntry[]>(
      `${BENCHMARK_API}/api/benchmarks/${modelId}/`
    );

    if (!entries) {
      models.push({ id: modelId, displayName, checkpoints: [] });
      continue;
    }

    const dirs = entries
      .filter((e) => e.type === "directory")
      .map((e) => e.name);

    const checkpoints: string[] = [];
    for (const dir of dirs) {
      const metrics = await fetchJson<Record<string, unknown>>(
        `${BENCHMARK_API}/api/benchmarks/${modelId}/${dir}/metrics.json`
      );
      if (metrics) checkpoints.push(dir);
    }

    checkpoints.sort((a, b) => {
      const numA = parseInt(a.replace(/\D/g, "")) || 0;
      const numB = parseInt(b.replace(/\D/g, "")) || 0;
      return numA - numB;
    });

    models.push({ id: modelId, displayName, checkpoints });
  }

  return models;
}

export async function getMetrics(modelId: string, checkpoint: string): Promise<MetricsFile | null> {
  const raw = await fetchJson<Record<string, unknown>>(
    `${BENCHMARK_API}/api/benchmarks/${modelId}/${checkpoint}/metrics.json`
  );
  if (!raw) return null;
  return migrateMetricsFile(raw);
}

export async function getSamples(modelId: string, checkpoint: string): Promise<SampleAnalysis[] | null> {
  const raw = await fetchJson<Record<string, unknown>[]>(
    `${BENCHMARK_API}/api/benchmarks/${modelId}/${checkpoint}/sample_analysis.json`
  );
  if (!raw) return null;
  return migrateSamples(raw);
}

export async function getErrorAnalysis(modelId: string, checkpoint: string): Promise<ErrorAnalysisFile | null> {
  return fetchJson<ErrorAnalysisFile>(
    `${BENCHMARK_API}/api/benchmarks/${modelId}/${checkpoint}/error_analysis.json`
  );
}


Now update the compare API to include MER:


     1→import { getMetrics, getModels } from "@/lib/data";
     2→import { LANGUAGES, LanguageMetrics } from "@/lib/types";
     3→
     4→export async function GET() {
     5→  const models = await getModels();
     6→  // { modelId: { ckpt: { lang: { wer_raw, wer_norm, wer_numcanon, cer_norm } } } }
     7→  const result: Record<string, Record<string, Record<string, Record<string, number>>>> = {};
     8→
     9→  for (const model of models) {
    10→    result[model.id] = {};
    11→    for (const ckpt of model.checkpoints) {
    12→      const metrics = await getMetrics(model.id, ckpt);
    13→      if (!metrics) continue;
    14→      const langMetrics: Record<string, Record<string, number>> = {};
    15→      for (const lang of LANGUAGES) {
    16→        const lm = metrics[lang] as LanguageMetrics | undefined;
    17→        if (lm && typeof lm === "object" && "wer_norm" in lm) {
    18→          langMetrics[lang] = {
    19→            wer_raw: lm.wer_raw,
    20→            wer_norm: lm.wer_norm,
    21→            wer_numcanon: lm.wer_numcanon,
    22→            cer_norm: lm.cer_norm,
    23→          };
    24→        }
    25→      }
    26→      langMetrics["__overall__"] = {
    27→        wer_raw: metrics.__overall__?.wer_raw ?? 0,
    28→        wer_norm: metrics.__overall__?.wer_norm ?? 0,
    29→        wer_numcanon: metrics.__overall__?.wer_numcanon ?? 0,
    30→        cer_norm: metrics.__overall__?.cer_norm ?? 0,
    31→      };
    32→      result[model.id][ckpt] = langMetrics;
    33→    }
    34→  }
    35→
    36→  return Response.json(result);
    37→}
    38→

import { getMetrics, getModels } from "@/lib/data";
import { LANGUAGES, LanguageMetrics } from "@/lib/types";

export async function GET() {
  const models = await getModels();
  // { modelId: { ckpt: { lang: { wer_raw, wer_norm, wer_numcanon, cer_norm } } } }
  const result: Record<string, Record<string, Record<string, Record<string, number>>>> = {};

  for (const model of models) {
    result[model.id] = {};
    for (const ckpt of model.checkpoints) {
      const metrics = await getMetrics(model.id, ckpt);
      if (!metrics) continue;
      const langMetrics: Record<string, Record<string, number>> = {};
      for (const lang of LANGUAGES) {
        const lm = metrics[lang] as LanguageMetrics | undefined;
        if (lm && typeof lm === "object" && "wer_norm" in lm) {
          langMetrics[lang] = {
            wer_raw: lm.wer_raw,
            wer_norm: lm.wer_norm,
            wer_numcanon: lm.wer_numcanon,
            cer_norm: lm.cer_norm,
          };
        }
      }
      langMetrics["__overall__"] = {
        wer_raw: metrics.__overall__?.wer_raw ?? 0,
        wer_norm: metrics.__overall__?.wer_norm ?? 0,
        wer_numcanon: metrics.__overall__?.wer_numcanon ?? 0,
        cer_norm: metrics.__overall__?.cer_norm ?? 0,
      };
      result[model.id][ckpt] = langMetrics;
    }
  }

  return Response.json(result);
}


Write

import { getMetrics, getModels } from "@/lib/data";
import { LANGUAGES, LanguageMetrics } from "@/lib/types";

export async function GET() {
  const models = await getModels();
  const result: Record<string, Record<string, Record<string, Record<string, number>>>> = {};

  for (const model of models) {
    result[model.id] = {};
    for (const ckpt of model.checkpoints) {
      const metrics = await getMetrics(model.id, ckpt);
      if (!metrics) continue;
      const langMetrics: Record<string, Record<string, number>> = {};
      for (const lang of LANGUAGES) {
        const lm = metrics[lang] as LanguageMetrics | undefined;
        if (lm && typeof lm === "object" && "wer_norm" in lm) {
          langMetrics[lang] = {
            wer_raw: lm.wer_raw,
            wer_norm: lm.wer_norm,
            wer_numcanon: lm.wer_numcanon,
            mer: lm.mer ?? 0,
            cer_norm: lm.cer_norm,
          };
        }
      }
      langMetrics["__overall__"] = {
        wer_raw: metrics.__overall__?.wer_raw ?? 0,
        wer_norm: metrics.__overall__?.wer_norm ?? 0,
        wer_numcanon: metrics.__overall__?.wer_numcanon ?? 0,
        mer: metrics.__overall__?.mer ?? 0,
        cer_norm: metrics.__overall__?.cer_norm ?? 0,
      };
      result[model.id][ckpt] = langMetrics;
    }
  }

  return Response.json(result);
}


The file /home/ubuntu/training/asr-benchmark-dashboard/src/app/api/compare/route.ts has been updated successfully.

import { getMetrics, getModels } from "@/lib/data";
import { LANGUAGES, LanguageMetrics } from "@/lib/types";

export async function GET() {
  const models = await getModels();
  const result: Record<string, Record<string, Record<string, Record<string, number>>>> = {};

  for (const model of models) {
    result[model.id] = {};
    for (const ckpt of model.checkpoints) {
      const metrics = await getMetrics(model.id, ckpt);
      if (!metrics) continue;
      const langMetrics: Record<string, Record<string, number>> = {};
      for (const lang of LANGUAGES) {
        const lm = metrics[lang] as LanguageMetrics | undefined;
        if (lm && typeof lm === "object" && "wer_norm" in lm) {
          langMetrics[lang] = {
            wer_raw: lm.wer_raw,
            wer_norm: lm.wer_norm,
            wer_numcanon: lm.wer_numcanon,
            mer: lm.mer ?? 0,
            cer_norm: lm.cer_norm,
          };
        }
      }
      langMetrics["__overall__"] = {
        wer_raw: metrics.__overall__?.wer_raw ?? 0,
        wer_norm: metrics.__overall__?.wer_norm ?? 0,
        wer_numcanon: metrics.__overall__?.wer_numcanon ?? 0,
        mer: metrics.__overall__?.mer ?? 0,
        cer_norm: metrics.__overall__?.cer_norm ?? 0,
      };
      result[model.id][ckpt] = langMetrics;
    }
  }

  return Response.json(result);
}

-  // { modelId: { ckpt: { lang: { wer_raw, wer_norm, wer_numcanon, cer_norm } } } }

+        mer: metrics.__overall__?.mer ?? 0,

import { getMetrics, getModels } from "@/lib/data";
import { LANGUAGES, LanguageMetrics } from "@/lib/types";

export async function GET() {
  const models = await getModels();
  // { modelId: { ckpt: { lang: { wer_raw, wer_norm, wer_numcanon, cer_norm } } } }
  const result: Record<string, Record<string, Record<string, Record<string, number>>>> = {};

  for (const model of models) {
    result[model.id] = {};
    for (const ckpt of model.checkpoints) {
      const metrics = await getMetrics(model.id, ckpt);
      if (!metrics) continue;
      const langMetrics: Record<string, Record<string, number>> = {};
      for (const lang of LANGUAGES) {
        const lm = metrics[lang] as LanguageMetrics | undefined;
        if (lm && typeof lm === "object" && "wer_norm" in lm) {
          langMetrics[lang] = {
            wer_raw: lm.wer_raw,
            wer_norm: lm.wer_norm,
            wer_numcanon: lm.wer_numcanon,
            cer_norm: lm.cer_norm,
          };
        }
      }
      langMetrics["__overall__"] = {
        wer_raw: metrics.__overall__?.wer_raw ?? 0,
        wer_norm: metrics.__overall__?.wer_norm ?? 0,
        wer_numcanon: metrics.__overall__?.wer_numcanon ?? 0,
        cer_norm: metrics.__overall__?.cer_norm ?? 0,
      };
      result[model.id][ckpt] = langMetrics;
    }
  }

  return Response.json(result);
}


Now update page.tsx — add MER to the metric cards and the detailed table. The metric selector already uses `METRIC_TIERS` so it'll pick up MER automatically. I just need to add the 5th metric card and add the spacing inflation strip: