The file /home/ubuntu/training/convert_to_schema_v1.py has been updated successfully.

#!/usr/bin/env python3
"""
Convert gemma3n-e2b benchmark predictions to BENCHMARK_SCHEMA v1.

Reads predictions.json from benchmark_results_gemma3n/<ckpt>/
Produces metrics.json, sample_analysis.json, error_analysis.json
in benchmark_outputs/gemma3n-e2b/<ckpt>/

Normalization tiers:
  raw:      NFC, trim whitespace
  norm:     NFKC, strip ZWJ/ZWNJ, collapse whitespace, lang-aware punct removal, case fold (Latin only)
  numcanon: norm + digit-grouping removal, Arabic numeral unification
"""

import json
import os
import re
import sys
import unicodedata
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path

import jiwer
import numpy as np

# ── Normalization Pipeline (versioned: v1) ──────────────────────────────

ZERO_WIDTH = re.compile(r"[\u200b-\u200f\u2028-\u202f\ufeff\u00ad]")
PUNCT_COMMON = re.compile(
    r"[!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~"
    r"\u0964\u0965"              # Devanagari danda / double danda
    r"\u2013\u2014\u2015"        # en-dash, em-dash, horizontal bar
    r"\u2018\u2019\u201c\u201d"  # curly quotes
    r"\u2026"                    # ellipsis
    r"\u00ab\u00bb"              # guillemets
    r"\u0022\u0027"              # straight quotes
    r"]"
)
DIGIT_GROUP_SEP = re.compile(r"(?<=\d)[,\s](?=\d)")

def norm_raw(text: str) -> str:
    """Tier 0: NFC + trim."""
    text = unicodedata.normalize("NFC", text)
    text = text.strip()
    return text

def norm_norm(text: str) -> str:
    """Tier 1: NFKC + strip ZW chars + collapse whitespace + remove punct + case fold."""
    text = unicodedata.normalize("NFKC", text)
    text = ZERO_WIDTH.sub("", text)
    text = re.sub(r"\s+", " ", text).strip()
    # Standardize punctuation variants before removal
    text = text.replace("\u2018", "'").replace("\u2019", "'")
    text = text.replace("\u201c", '"').replace("\u201d", '"')
    text = text.replace("\u2013", "-").replace("\u2014", "-")
    text = text.replace("\u0965", "\u0964")  # double danda → single
    # Remove punctuation
    text = PUNCT_COMMON.sub("", text)
    text = re.sub(r"\s+", " ", text).strip()
    # Case fold (only affects Latin script characters)
    text = text.lower()
    return text

def norm_numcanon(text: str) -> str:
    """Tier 2: norm + number canonicalization."""
    text = norm_norm(text)
    # Remove digit grouping separators: "25,000" → "25000", "1 000" → "1000"
    text = DIGIT_GROUP_SEP.sub("", text)
    return text

# ── Per-sample WER ──────────────────────────────────────────────────────

def sample_wer(ref: str, hyp: str) -> float:
    """Compute WER for a single sample. Returns 0-100 float."""
    if not ref.strip():
        return 0.0 if not hyp.strip() else 100.0
    try:
        return round(jiwer.wer(ref, hyp) * 100, 2)
    except Exception:
        return 100.0

# ── Flag Detection ──────────────────────────────────────────────────────

def detect_flags(ref_raw, hyp_raw, ref_n, hyp_n, ref_nc, hyp_nc) -> list:
    flags = []
    if ref_raw == hyp_raw:
        flags.append("exact_match")
    if ref_n == hyp_n:
        flags.append("exact_match_norm")
    if not hyp_raw.strip():
        flags.append("empty_hypothesis")
    # Punctuation-only diff: raw differs but norm matches
    if ref_raw != hyp_raw and ref_n == hyp_n:
        flags.append("punctuation_only_diff")
    # Numeric mismatch: numcanon changes the match status
    if ref_n != hyp_n and ref_nc == hyp_nc:
        flags.append("numeric_mismatch")
    # High WER
    w = sample_wer(ref_n, hyp_n)
    if w > 80:
        flags.append("high_wer")
    return flags

# ── Error Analysis (alignment-based) ────────────────────────────────────

def compute_error_ops(refs: list, hyps: list):
    """Compute word-level substitutions, insertions, deletions using jiwer alignment."""
    subs = Counter()
    ins = Counter()
    dels = Counter()

    for ref, hyp in zip(refs, hyps):
        ref_words = ref.split()
        hyp_words = hyp.split()
        try:
            out = jiwer.process_words(ref, hyp)
            for chunk in out.alignments[0]:
                if chunk.type == "substitute":
                    for ri, hi in zip(
                        range(chunk.ref_start_idx, chunk.ref_end_idx),
                        range(chunk.hyp_start_idx, chunk.hyp_end_idx),
                    ):
                        if ri < len(ref_words) and hi < len(hyp_words):
                            subs[(ref_words[ri], hyp_words[hi])] += 1
                elif chunk.type == "insert":
                    for hi in range(chunk.hyp_start_idx, chunk.hyp_end_idx):
                        if hi < len(hyp_words):
                            ins[hyp_words[hi]] += 1
                elif chunk.type == "delete":
                    for ri in range(chunk.ref_start_idx, chunk.ref_end_idx):
                        if ri < len(ref_words):
                            dels[ref_words[ri]] += 1
        except Exception:
            continue

    return subs, ins, dels

def build_error_analysis(predictions: list) -> dict:
    """Build per-language error analysis."""
    by_lang = defaultdict(lambda: {"refs": [], "hyps": [], "ids": [], "wers": []})

    for p in predictions:
        lang = p["language"]
        ref_n = norm_norm(p["reference"])
        hyp_n = norm_norm(p.get("hypothesis", ""))
        w = sample_wer(ref_n, hyp_n)
        by_lang[lang]["refs"].append(ref_n)
        by_lang[lang]["hyps"].append(hyp_n)
        by_lang[lang]["ids"].append(p["id"])
        by_lang[lang]["wers"].append(w)

    analysis = {}
    all_lang_wers = {}

    for lang in sorted(by_lang.keys()):
        d = by_lang[lang]
        subs, ins, dels = compute_error_ops(d["refs"], d["hyps"])

        top_subs = [{"ref": r, "hyp": h, "count": c} for (r, h), c in subs.most_common(20)]
        top_ins = [{"word": w, "count": c} for w, c in ins.most_common(20)]
        top_dels = [{"word": w, "count": c} for w, c in dels.most_common(20)]

        # Error buckets
        empty_count = sum(1 for h in d["hyps"] if not h.strip())
        punct_only = 0
        numeric_mm = 0
        for p in predictions:
            if p["language"] != lang:
                continue
            ref_raw = norm_raw(p["reference"])
            hyp_raw = norm_raw(p.get("hypothesis", ""))
            ref_n = norm_norm(p["reference"])
            hyp_n = norm_norm(p.get("hypothesis", ""))
            ref_nc = norm_numcanon(p["reference"])
            hyp_nc = norm_numcanon(p.get("hypothesis", ""))
            if ref_raw != hyp_raw and ref_n == hyp_n:
                punct_only += 1
            if ref_n != hyp_n and ref_nc == hyp_nc:
                numeric_mm += 1

        # Best/worst samples by WER
        sorted_idx = np.argsort(d["wers"])
        worst_ids = [d["ids"][i] for i in sorted_idx[-3:]][::-1]
        best_ids = [d["ids"][i] for i in sorted_idx[:3]]

        analysis[lang] = {
            "top_substitutions": top_subs,
            "top_insertions": top_ins,
            "top_deletions": top_dels,
            "error_buckets": {
                "numeric_mismatch_count": numeric_mm,
                "punctuation_only_count": punct_only,
                "spacing_tokenization_count": 0,
                "entity_mismatch_count": 0,
                "script_confusion_count": 0,
                "empty_hypothesis_count": empty_count,
            },
            "examples": {
                "worst_samples": worst_ids,
                "best_samples": best_ids,
            },
        }
        all_lang_wers[lang] = np.mean(d["wers"])

    # Summary
    sorted_langs = sorted(all_lang_wers, key=all_lang_wers.get)
    best_langs = sorted_langs[:3]
    worst_langs = sorted_langs[-3:][::-1]

    # Determine diagnosis
    # Compare overall raw vs norm delta
    analysis["__summary__"] = {
        "model_diagnosis": "recognition-limited",
        "primary_error_source": "recognition",
        "numeric_verbalization_impact": "low",
        "formatting_impact": "low",
        "worst_languages": worst_langs,
        "best_languages": best_langs,
    }

    return analysis

# ── Main Conversion ─────────────────────────────────────────────────────

def convert_checkpoint(src_dir: str, dst_dir: str, ckpt_name: str):
    """Convert one checkpoint's predictions to schema v1."""
    os.makedirs(dst_dir, exist_ok=True)

    # Load predictions
    pred_file = os.path.join(src_dir, "predictions.json")
    with open(pred_file) as f:
        predictions = json.load(f)
    print(f"  Loaded {len(predictions)} predictions from {pred_file}")

    # Load old metrics for meta info
    old_meta_file = os.path.join(src_dir, "metrics.json")
    with open(old_meta_file) as f:
        old_metrics = json.load(f)
    old_meta = old_metrics.get("__meta__", {})

    # ── Compute all normalized texts ────────────────────────────────
    by_lang = defaultdict(lambda: {
        "refs_raw": [], "hyps_raw": [],
        "refs_norm": [], "hyps_norm": [],
        "refs_nc": [], "hyps_nc": [],
    })

    sample_analysis = []

    for p in predictions:
        lang = p["language"]
        ref = p["reference"]
        hyp = p.get("hypothesis", "")

        r_raw = norm_raw(ref)
        h_raw = norm_raw(hyp)
        r_norm = norm_norm(ref)
        h_norm = norm_norm(hyp)
        r_nc = norm_numcanon(ref)
        h_nc = norm_numcanon(hyp)

        by_lang[lang]["refs_raw"].append(r_raw)
        by_lang[lang]["hyps_raw"].append(h_raw)
        by_lang[lang]["refs_norm"].append(r_norm)
        by_lang[lang]["hyps_norm"].append(h_norm)
        by_lang[lang]["refs_nc"].append(r_nc)
        by_lang[lang]["hyps_nc"].append(h_nc)

        flags = detect_flags(r_raw, h_raw, r_norm, h_norm, r_nc, h_nc)
        w_raw = sample_wer(r_raw, h_raw)
        w_norm = sample_wer(r_norm, h_norm)

        sample_analysis.append({
            "id": p["id"],
            "language": lang,
            "reference": ref,
            "hypothesis": hyp,
            "ref_norm": r_norm,
            "hyp_norm": h_norm,
            "ref_numcanon": r_nc,
            "hyp_numcanon": h_nc,
            "wer_raw": w_raw,
            "wer_norm": w_norm,
            "flags": flags,
        })

    # ── Compute metrics per language ────────────────────────────────
    metrics = {}
    all_refs_raw, all_hyps_raw = [], []
    all_refs_norm, all_hyps_norm = [], []
    all_refs_nc, all_hyps_nc = [], []

    for lang in sorted(by_lang.keys()):
        d = by_lang[lang]
        n = len(d["refs_raw"])

        # Skip empty refs
        valid = [(i, r) for i, r in enumerate(d["refs_norm"]) if r.strip()]
        if not valid:
            continue

        w_raw = round(jiwer.wer(d["refs_raw"], d["hyps_raw"]) * 100, 2)
        w_norm = round(jiwer.wer(d["refs_norm"], d["hyps_norm"]) * 100, 2)
        w_nc = round(jiwer.wer(d["refs_nc"], d["hyps_nc"]) * 100, 2)
        c_norm = round(jiwer.cer(d["refs_norm"], d["hyps_norm"]) * 100, 2)

        empty_count = sum(1 for h in d["hyps_raw"] if not h.strip())

        metrics[lang] = {
            "n_samples": n,
            "wer_raw": w_raw,
            "wer_norm": w_norm,
            "wer_numcanon": w_nc,
            "cer_norm": c_norm,
            "empty_hypotheses": empty_count,
            "normalization_delta": {
                "raw_to_norm": round(w_raw - w_norm, 2),
                "norm_to_numcanon": round(w_norm - w_nc, 2),
            },
        }

        all_refs_raw.extend(d["refs_raw"])
        all_hyps_raw.extend(d["hyps_raw"])
        all_refs_norm.extend(d["refs_norm"])
        all_hyps_norm.extend(d["hyps_norm"])
        all_refs_nc.extend(d["refs_nc"])
        all_hyps_nc.extend(d["hyps_nc"])

    # ── Aggregates ──────────────────────────────────────────────────
    metrics["__overall__"] = {
        "n_samples": len(all_refs_raw),
        "wer_raw": round(jiwer.wer(all_refs_raw, all_hyps_raw) * 100, 2),
        "wer_norm": round(jiwer.wer(all_refs_norm, all_hyps_norm) * 100, 2),
        "wer_numcanon": round(jiwer.wer(all_refs_nc, all_hyps_nc) * 100, 2),
        "cer_norm": round(jiwer.cer(all_refs_norm, all_hyps_norm) * 100, 2),
    }

    lang_keys = [k for k in metrics if not k.startswith("__")]
    metrics["__macro_avg__"] = {
        "n_languages": len(lang_keys),
        "wer_raw": round(np.mean([metrics[k]["wer_raw"] for k in lang_keys]), 2),
        "wer_norm": round(np.mean([metrics[k]["wer_norm"] for k in lang_keys]), 2),
        "wer_numcanon": round(np.mean([metrics[k]["wer_numcanon"] for k in lang_keys]), 2),
        "cer_norm": round(np.mean([metrics[k]["cer_norm"] for k in lang_keys]), 2),
    }

    # ── Meta ────────────────────────────────────────────────────────
    metrics["__meta__"] = {
        "checkpoint": old_meta.get("checkpoint", ""),
        "checkpoint_name": ckpt_name,
        "model_id": "gemma3n-e2b",
        "model_type": old_meta.get("model_type", "gemma3n-E2B-asr"),
        "dataset": old_meta.get("dataset", "BayAreaBoys/indic-asr-benchmark-6k"),
        "batch_size": old_meta.get("batch_size", 128),
        "inference_time_sec": old_meta.get("inference_time_sec", 0),
        "total_audio_sec": old_meta.get("total_audio_sec", 0),
        "rtf": old_meta.get("rtf", 0),
        "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "gpu": "NVIDIA A100-SXM4-80GB",
        "framework": "transformers",
        "normalization_version": "v1",
        "jiwer_version": jiwer.__version__,
    }

    # ── Update error analysis summary with actual deltas ────────────
    ov = metrics["__overall__"]
    raw_norm_delta = ov["wer_raw"] - ov["wer_norm"]
    norm_nc_delta = ov["wer_norm"] - ov["wer_numcanon"]

    # ── Write metrics.json ──────────────────────────────────────────
    with open(os.path.join(dst_dir, "metrics.json"), "w") as f:
        json.dump(metrics, f, indent=2, ensure_ascii=False)
    print(f"  metrics.json: {len(lang_keys)} langs, overall wer_norm={ov['wer_norm']}%")

    # ── Write sample_analysis.json ──────────────────────────────────
    with open(os.path.join(dst_dir, "sample_analysis.json"), "w") as f:
        json.dump(sample_analysis, f, indent=2, ensure_ascii=False)
    print(f"  sample_analysis.json: {len(sample_analysis)} samples")

    # ── Build and write error_analysis.json ─────────────────────────
    error_analysis = build_error_analysis(predictions)

    # Update summary with actual metric deltas
    fmt_impact = "high" if raw_norm_delta > 10 else "moderate" if raw_norm_delta > 5 else "low"
    num_impact = "high" if norm_nc_delta > 5 else "moderate" if norm_nc_delta > 2 else "low"

    if raw_norm_delta > 10:
        diagnosis = "formatting-limited"
        primary = "formatting"
    elif norm_nc_delta > 5:
        diagnosis = "numeric-limited"
        primary = "numeric_verbalization"
    else:
        diagnosis = "recognition-limited"
        primary = "recognition"

    error_analysis["__summary__"]["model_diagnosis"] = diagnosis
    error_analysis["__summary__"]["primary_error_source"] = primary
    error_analysis["__summary__"]["formatting_impact"] = fmt_impact
    error_analysis["__summary__"]["numeric_verbalization_impact"] = num_impact

    with open(os.path.join(dst_dir, "error_analysis.json"), "w") as f:
        json.dump(error_analysis, f, indent=2, ensure_ascii=False)
    print(f"  error_analysis.json: {len(lang_keys)} langs, diagnosis={diagnosis}")

    return metrics

def main():
    src_base = "/home/ubuntu/training/benchmark_results_gemma3n"
    dst_base = "/home/ubuntu/training/benchmark_outputs/gemma3n-e2b"

    for ckpt in ["ckpt-10000", "ckpt-20000"]:
        src = os.path.join(src_base, ckpt)
        dst = os.path.join(dst_base, ckpt)
        print(f"\n{'='*60}")
        print(f"Converting {ckpt}")
        print(f"{'='*60}")
        metrics = convert_checkpoint(src, dst, ckpt)

        # Print summary table
        ov = metrics["__overall__"]
        ma = metrics["__macro_avg__"]
        print(f"\n  {'Language':<14} {'wer_raw':>8} {'wer_norm':>9} {'wer_nc':>8} {'cer_norm':>9} {'Δraw→norm':>10} {'Δnorm→nc':>10}")
        print(f"  {'-'*14} {'-'*8} {'-'*9} {'-'*8} {'-'*9} {'-'*10} {'-'*10}")
        for lang in sorted(k for k in metrics if not k.startswith("__")):
            m = metrics[lang]
            d = m["normalization_delta"]
            print(f"  {lang:<14} {m['wer_raw']:>8.2f} {m['wer_norm']:>9.2f} {m['wer_numcanon']:>8.2f} "
                  f"{m['cer_norm']:>9.2f} {d['raw_to_norm']:>+10.2f} {d['norm_to_numcanon']:>+10.2f}")
        print(f"  {'-'*14} {'-'*8} {'-'*9} {'-'*8} {'-'*9} {'-'*10} {'-'*10}")
        print(f"  {'OVERALL':<14} {ov['wer_raw']:>8.2f} {ov['wer_norm']:>9.2f} {ov['wer_numcanon']:>8.2f} {ov['cer_norm']:>9.2f}")
        print(f"  {'MACRO-AVG':<14} {ma['wer_raw']:>8.2f} {ma['wer_norm']:>9.2f} {ma['wer_numcanon']:>8.2f} {ma['cer_norm']:>9.2f}")

if __name__ == "__main__":
    main()

-        "jiwer_version": jiwer.__version__,

+        "jiwer_version": getattr(jiwer, "__version__", "unknown"),


Bash

Run schema v1 conversion


============================================================
Converting ckpt-10000
============================================================
  Loaded 6000 predictions from /home/ubuntu/training/benchmark_results_gemma3n/ckpt-10000/predictions.json
  metrics.json: 12 langs, overall wer_norm=36.24%
  sample_analysis.json: 6000 samples
  error_analysis.json: 12 langs, diagnosis=recognition-limited

  Language        wer_raw  wer_norm   wer_nc  cer_norm  Δraw→norm   Δnorm→nc
  -------------- -------- --------- -------- --------- ---------- ----------
  assamese          36.32     36.13    36.13     18.50      +0.19      +0.00
  bengali           33.59     33.10    33.10     15.47      +0.49      +0.00
  english           27.43     15.11    15.13      9.45     +12.32      -0.02
  gujarati          21.98     21.98    21.98      6.97      +0.00      +0.00
  hindi             14.81     14.81    14.81      5.48      +0.00      +0.00
  kannada           42.60     42.58    42.58     15.18      +0.02      +0.00
  malayalam         45.49     45.42    45.42     10.69      +0.07      +0.00
  marathi           35.05     35.05    35.05     12.48      +0.00      +0.00
  odia              82.84     82.51    76.01     55.14      +0.33      +6.50
  punjabi           31.53     31.13    31.13     37.30      +0.40      +0.00
  tamil             47.93     47.89    47.89     17.72      +0.04      +0.00
  telugu            31.79     31.79    31.01      9.57      +0.00      +0.78
  -------------- -------- --------- -------- --------- ---------- ----------
  OVERALL           37.33     36.24    35.66     17.38
  MACRO-AVG         37.61     36.46    35.85     17.83

============================================================
Converting ckpt-20000
============================================================
  Loaded 6000 predictions from /home/ubuntu/training/benchmark_results_gemma3n/ckpt-20000/predictions.json
  metrics.json: 12 langs, overall wer_norm=33.95%
  sample_analysis.json: 6000 samples
  error_analysis.json: 12 langs, diagnosis=recognition-limited

  Language        wer_raw  wer_norm   wer_nc  cer_norm  Δraw→norm   Δnorm→nc
  -------------- -------- --------- -------- --------- ---------- ----------
  assamese          37.61     37.46    37.46     19.35      +0.15      +0.00
  bengali           26.49     26.03    26.03      9.44      +0.46      +0.00
  english           29.70     17.94    17.96     12.20     +11.76      -0.02
  gujarati          22.27     22.27    22.27      7.54      +0.00      +0.00
  hindi             14.54     14.54    14.54      4.75      +0.00      +0.00
  kannada           43.49     43.47    42.73     17.23      +0.02      +0.74
  malayalam         44.70     44.67    44.67     10.55      +0.03      +0.00
  marathi           30.53     30.51    30.51      9.28      +0.02      +0.00
  odia              73.19     73.08    72.97     51.38      +0.11      +0.11
  punjabi           27.07     26.69    26.69     13.96      +0.38      +0.00
  tamil             41.38     41.38    41.38     16.75      +0.00      +0.00
  telugu            30.62     30.62    30.62      9.57      +0.00      +0.00
  -------------- -------- --------- -------- --------- ---------- ----------
  OVERALL           34.97     33.95    33.89     15.25
  MACRO-AVG         35.13     34.06    33.99     15.17


All 3 files generated per checkpoint. Let me verify the outputs match the schema.


python3 -c "
import json

for ckpt in ['ckpt-10000', 'ckpt-20000']:
    base = f'/home/ubuntu/training/benchmark_outputs/gemma3n-e2b/{ckpt}'

    # Verify metrics.json
    with open(f'{base}/metrics.json') as f:
        m = json.load(f)
    lang_keys = [k for k in m if not k.startswith('__')]
    meta = m['__meta__']
    hindi = m['hindi']
    print(f'{ckpt} metrics.json:')
    print(f'  langs: {len(lang_keys)}, per-lang keys: {sorted(hindi.keys())}')
    print(f'  overall keys: {sorted(m[\"__overall__\"].keys())}')
    print(f'  meta.model_id={meta[\"model_id\"]}, norm_version={meta[\"normalization_version\"]}')
    print(f'  hindi.normalization_delta: {hindi[\"normalization_delta\"]}')

    # Verify sample_analysis.json
    with open(f'{base}/sample_analysis.json') as f:
        s = json.load(f)
    print(f'  sample_analysis: {len(s)} samples, keys: {sorted(s[0].keys())}')
    print(f'  sample flags example: {s[0].get(\"flags\", [])}')

    # Verify error_analysis.json
    with open(f'{base}/error_analysis.json') as f:
        e = json.load(f)
    elang_keys = [k for k in e if not k.startswith('__')]
    summary = e['__summary__']
    hindi_err = e['hindi']
    print(f'  error_analysis: {len(elang_keys)} langs')
    print(f'  hindi top_subs: {len(hindi_err[\"top_substitutions\"])}, top_ins: {len(hindi_err[\"top_insertions\"])}, top_dels: {len(hindi_err[\"top_deletions\"])}')
    print(f'  summary: diagnosis={summary[\"model_diagnosis\"]}, worst={summary[\"worst_languages\"]}, best={summary[\"best_languages\"]}')
    print()
" 2>&1

Verify schema compliance


ckpt-10000 metrics.json:
  langs: 12, per-lang keys: ['cer_norm', 'empty_hypotheses', 'n_samples', 'normalization_delta', 'wer_norm', 'wer_numcanon', 'wer_raw']
  overall keys: ['cer_norm', 'n_samples', 'wer_norm', 'wer_numcanon', 'wer_raw']
  meta.model_id=gemma3n-e2b, norm_version=v1
  hindi.normalization_delta: {'raw_to_norm': 0.0, 'norm_to_numcanon': 0.0}
  sample_analysis: 6000 samples, keys: ['flags', 'hyp_norm', 'hyp_numcanon', 'hypothesis', 'id', 'language', 'ref_norm', 'ref_numcanon', 'reference', 'wer_norm', 'wer_raw']
  sample flags example: []
  error_analysis: 12 langs
  hindi top_subs: 20, top_ins: 20, top_dels: 20
  summary: diagnosis=recognition-limited, worst=['odia', 'tamil', 'malayalam'], best=['hindi', 'english', 'gujarati']

ckpt-20000 metrics.json:
  langs: 12, per-lang keys: ['cer_norm', 'empty_hypotheses', 'n_samples', 'normalization_delta', 'wer_norm', 'wer_numcanon', 'wer_raw']
  overall keys: ['cer_norm', 'n_samples', 'wer_norm', 'wer_numcanon', 'wer_raw']
  meta.model_id=gemma3n-e2b, norm_version=v1
  hindi.normalization_delta: {'raw_to_norm': 0.0, 'norm_to_numcanon': 0.0}
  sample_analysis: 6000 samples, keys: ['flags', 'hyp_norm', 'hyp_numcanon', 'hypothesis', 'id', 'language', 'ref_norm', 'ref_numcanon', 'reference', 'wer_norm', 'wer_raw']
  sample flags example: []
  error_analysis: 12 langs
  hindi top_subs: 20, top_ins: 20, top_dels: 20
  summary: diagnosis=recognition-limited, worst=['odia', 'malayalam', 'kannada'], best=['hindi', 'gujarati', 'bengali']


ls -lh /home/ubuntu/training/benchmark_outputs/gemma3n-e2b/ckpt-*/  2>&1

Final file listing


/home/ubuntu/training/benchmark_outputs/gemma3n-e2b/ckpt-10000/:
total 8.7M
-rw-rw-r-- 1 ubuntu ubuntu  64K Mar 26 20:37 error_analysis.json
-rw-rw-r-x 1 ubuntu ubuntu 3.8K Mar 26 20:37 metrics.json
-rw-rw-r-x 1 ubuntu ubuntu 8.6M Mar 26 20:37 sample_analysis.json

/home/ubuntu/training/benchmark_outputs/gemma3n-e2b/ckpt-20000/:
total 8.6M
-rw-rw-r-- 1 ubuntu ubuntu  64K Mar 26 20:37 error_analysis.json
-rw-rw-r-x 1 ubuntu ubuntu 3.8K Mar 26 20:37 metrics.json
-rw-rw-r-x 1 ubuntu ubuntu 8.5M Mar 26 20:37 sample_analysis.json


All files generated and schema-compliant. Here's the summary:

**Files placed at `/home/ubuntu/training/benchmark_outputs/gemma3n-e2b/`:**

```
ckpt-10000/
  metrics.json         (3.8K)  — 4 metric tiers + normalization_delta + v1 meta
  sample_analysis.json (8.6M)  — 6000 samples with ref_norm/hyp_norm/ref_numcanon/hyp_numcanon + flags
  error_analysis.json  (64K)   — top 20 subs/ins/dels per lang + error_buckets + __summary__

ckpt-20000/
  metrics.json         (3.8K)
  sample_analysis.json (8.5M)
  error_analysis.json  (64K)
```

**Schema v1 compliance:**
- `wer_raw`, `wer_norm`, `wer_numcanon`, `cer_norm` — all 4 tiers
- `normalization_delta` per language (`raw_to_norm`, `norm_to_numcanon`)
- `__meta__`: `model_id=gemma3n-e2b`, `normalization_version=v1`, `timestamp`, `gpu`, `framework`
- `sample_analysis`: per-sample `ref_norm`/`hyp_norm`/`ref_numcanon`/`hyp_numcanon` + `wer_raw`/`wer_norm` + `flags`
- `error_analysis`: top subs/ins/dels, `error_buckets`, `__summary__` with `model_diagnosis=recognition-limited`

**Key insight from the new tiers:** English has the biggest raw→norm delta (+12.3 WER points) due to casing/punctuation. Odia shows the biggest numcanon delta (+6.5 points at ckpt-10000), suggesting numeric handling issues. The model is **recognition-limited** overall — normalization gains are small for most Indic languages.


❯    You are an expert ASR evaluation agent for multilingual Indic speech recognition.…


so what happened is, i made a few changes with the training, and just was training with different data. so there are 2 new ckpts of gemma3n. they're 20k and 40k again. can you first cleanup the local, previous data, then  pull from R2 these 20k and 40k ckpts, and re-run eval and benchmarks on those ckpts and also update it in dashboard's JSON area. check @benchmark_schema/BENCHMARK_SCHEMA.md and there has been some commits to git from the repo, make sure to pull for latest code changes. and lmk if you have any questions.


Let me check R2 for the new checkpoints, pull git changes, and clean up old data — all in parallel.


source /home/ubuntu/training/.env && AWS_ACCESS_KEY_ID=$R2_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$R2_SECRET_ACCESS_KEY aws s3 ls "s3://ptcheckpoints/gemma3n-E2B-asr/" --endpoint-url "$R2_ENDPOINT_URL" --region auto 2>&1

Check R2 for gemma3n checkpoint dates


cd /home/ubuntu/training/gemma3n-E2B-asr-ft && git pull 2>&1

Pull latest git changes


                           PRE 03-24-2026/
                           PRE 03-28-2026/


ls -la /home/ubuntu/training/checkpoints/ 2>&1

Check current local checkpoints


From https://github.com/Maya-AI-ML-Trainings-Cookbooks/gemma3n-E2B-asr-ft
   d70c21e..6010413  main       -> origin/main
Updating d70c21e..6010413
Fast-forward
 artifacts/phase4/flagged_repetitive.csv            | 181 +++++++++
 artifacts/phase4/rebuild_stats.json                |  15 +-
 artifacts/phase6_preflight.json                    |   2 +-
 configs/fsdp2_config.yaml                          |  24 ++
 configs/fsdp_config.yaml                           |  24 ++
 configs/gemma3n_asr_fsdp2.yaml                     |  71 ++++
 configs/gemma3n_asr_full_ft.yaml                   |   1 +
 configs/smoke_fsdp_profile.yaml                    |  54 +++
 scripts/gpu_watchdog.sh                            |  78 ++++
 scripts/launch_fsdp2.sh                            |  62 ++++
 scripts/launch_fsdp2_v2.sh                         |  48 +++
 scripts/oom_recovery.sh                            |  58 +++
 scripts/profile_fsdp.sh                            |  34 ++
 scripts/push_ckpt_r2.sh                            |   2 +-
 scripts/resume_full_ft.sh                          |  64 ++++
 tools/drop_flagged_rows.py                         |  45 +++
 tools/find_repetitive_transcripts.py               |  97 +++++
 tools/phase4_rebuild_mixed_tempsample.py           | 405 +++++++++++++++++++++
 .../src/llamafactory/model/model_utils/visual.py   |   2 +-
 19 files changed, 1260 insertions(+), 7 deletions(-)
 create mode 100644 artifacts/phase4/flagged_repetitive.csv
 create mode 100644 configs/fsdp2_config.yaml
 create mode 100644 configs/fsdp_config.yaml
 create mode 100644 configs/gemma3n_asr_fsdp2.yaml
 create mode 100644 configs/smoke_fsdp_profile.yaml
 create mode 100755 scripts/gpu_watchdog.sh
 create mode 100755 scripts/launch_fsdp2.sh
 create mode 100755 scripts/launch_fsdp2_v2.sh
 create mode 100755 scripts/oom_recovery.sh
 create mode 100755 scripts/profile_fsdp.sh
 create mode 100755 scripts/resume_full_ft.sh
 create mode 100644 tools/drop_flagged_rows.py
 create mode 100644 tools/find_repetitive_transcripts.py
 create mode 100644 tools/phase4_rebuild_mixed_tempsample.py


total 84
drwxrwxr-x 21 ubuntu ubuntu 4096 Mar 29 07:08 .
drwxrwxr-x 18 ubuntu ubuntu 4096 Mar 29 06:45 ..
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 26 00:59 gemma3n-e2b-ckpt-10000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 26 01:07 gemma3n-e2b-ckpt-20000
drwxrwxr-x  4 ubuntu ubuntu 4096 Mar 25 18:43 llama3.2-1b-tts
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 27 22:33 maya-asr-tdt-1.1b-ckpt-120000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 27 07:16 maya-asr-tdt-1.1b-ckpt-20000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 29 06:45 maya-asr-tdt-1.1b-ckpt-200000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 27 07:17 maya-asr-tdt-1.1b-ckpt-40000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 27 05:28 maya-asr-tdt-1.1b-ckpt-60000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 27 07:22 maya-asr-tdt-1.1b-ckpt-80000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 25 23:02 qwen3-asr
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 26 02:30 qwen3-asr-ckpt-100000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 26 19:35 qwen3-asr-ckpt-170000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 27 02:50 qwen3-asr-ckpt-200000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 25 21:23 qwen3-asr-ckpt-24000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 27 19:30 qwen3-asr-ckpt-250000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 28 06:35 qwen3-asr-ckpt-300000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 29 07:09 qwen3-asr-ckpt-400000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 25 21:21 qwen3-asr-ckpt-72000
drwxrwxr-x  2 ubuntu ubuntu 4096 Mar 25 03:16 xcodec2


New date folder `03-28-2026` on R2. Let me check what's in it, then clean up old gemma3n checkpoints and download the new ones.


source /home/ubuntu/training/.env && AWS_ACCESS_KEY_ID=$R2_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$R2_SECRET_ACCESS_KEY aws s3 ls "s3://ptcheckpoints/gemma3n-E2B-asr/03-28-2026/" --endpoint-url "$R2_ENDPOINT_URL" --region auto 2>&1

List new checkpoints in R2


                           PRE ckpt-20000/
                           PRE ckpt-40000/

                           PRE ckpt-20000/
                           PRE ckpt-40000/


Both ckpt-20000 and ckpt-40000 confirmed. Let me clean up old gemma3n checkpoints and start downloading new ones.


# Clean up old gemma3n checkpoints
rm -rf /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-10000 /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000 2>&1 && echo "Old checkpoints cleaned"
# Clean up old benchmark outputs
rm -rf /home/ubuntu/training/benchmark_outputs/gemma3n-e2b/ckpt-10000 /home/ubuntu/training/benchmark_outputs/gemma3n-e2b/ckpt-20000 2>&1 && echo "Old benchmark outputs cleaned"
# Clean up old raw results
rm -rf /home/ubuntu/training/benchmark_results_gemma3n/ckpt-10000 /home/ubuntu/training/benchmark_results_gemma3n/ckpt-20000 /home/ubuntu/training/benchmark_results_gemma3n/ckpt-10000-smoke 2>&1 && echo "Old raw results cleaned"

Clean up old gemma3n data


Old checkpoints cleaned
Old benchmark outputs cleaned
Old raw results cleaned


Now download both new checkpoints (model files only, skip optimizer).


source /home/ubuntu/training/.env && mkdir -p /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000 && for f in chat_template.jinja config.json generation_config.json model.safetensors processor_config.json tokenizer.json tokenizer_config.json; do
  AWS_ACCESS_KEY_ID=$R2_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$R2_SECRET_ACCESS_KEY aws s3 cp "s3://ptcheckpoints/gemma3n-E2B-asr/03-28-2026/ckpt-20000/$f" "/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/$f" --endpoint-url "$R2_ENDPOINT_URL" --region auto &
done
wait && echo "ckpt-20000 DONE"

Download ckpt-20000 model files


Command running in background with ID: bcy99cwa3. Output is being written to: /tmp/claude-1001/-home-ubuntu-training/5e207b1e-98a7-43a4-9947-9579680cf6bd/tasks/bcy99cwa3.output


source /home/ubuntu/training/.env && mkdir -p /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000 && for f in chat_template.jinja config.json generation_config.json model.safetensors processor_config.json tokenizer.json tokenizer_config.json; do
  AWS_ACCESS_KEY_ID=$R2_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$R2_SECRET_ACCESS_KEY aws s3 cp "s3://ptcheckpoints/gemma3n-E2B-asr/03-28-2026/ckpt-40000/$f" "/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/$f" --endpoint-url "$R2_ENDPOINT_URL" --region auto &
done
wait && echo "ckpt-40000 DONE"

Download ckpt-40000 model files


Command running in background with ID: bosyl0g8u. Output is being written to: /tmp/claude-1001/-home-ubuntu-training/5e207b1e-98a7-43a4-9947-9579680cf6bd/tasks/bosyl0g8u.output


Both downloading in parallel. Meanwhile let me update the benchmark script to include the new schema metrics (`space_norm_wer`, `mer`) and the conversion script.


1	#!/usr/bin/env python3
2	"""
3	Convert gemma3n-e2b benchmark predictions to BENCHMARK_SCHEMA v1.
4
5	Reads predictions.json from benchmark_results_gemma3n/<ckpt>/
6	Produces metrics.json, sample_analysis.json, error_analysis.json
7	in benchmark_outputs/gemma3n-e2b/<ckpt>/
8
9	Normalization tiers:
10	  raw:      NFC, trim whitespace
11	  norm:     NFKC, strip ZWJ/ZWNJ, collapse whitespace, lang-aware punct removal, case fold (Latin only)
12	  numcanon: norm + digit-grouping removal, Arabic numeral unification
13	"""
14
15	import json
16	import os
17	import re
18	import sys
19	import unicodedata
20	from collections import Counter, defaultdict

#!/usr/bin/env python3
"""
Convert gemma3n-e2b benchmark predictions to BENCHMARK_SCHEMA v1.

Reads predictions.json from benchmark_results_gemma3n/<ckpt>/
Produces metrics.json, sample_analysis.json, error_analysis.json
in benchmark_outputs/gemma3n-e2b/<ckpt>/

Normalization tiers:
  raw:      NFC, trim whitespace
  norm:     NFKC, strip ZWJ/ZWNJ, collapse whitespace, lang-aware punct removal, case fold (Latin only)
  numcanon: norm + digit-grouping removal, Arabic numeral unification
"""

import json
import os
import re
import sys
import unicodedata
from collections import Counter, defaultdict


The schema now needs `space_norm_wer` and `mer` in addition to what we have. Let me rewrite the conversion script to be a standalone pipeline that reads predictions and outputs all 3 schema files with all 6 metric tiers.


Write

#!/usr/bin/env python3
"""
Generate BENCHMARK_SCHEMA v1 outputs for gemma3n-e2b checkpoints.

Reads predictions.json → produces metrics.json, sample_analysis.json, error_analysis.json
with all 6 metric tiers: wer_raw, wer_norm, wer_numcanon, space_norm_wer, mer, cer_norm.

Usage:
    python generate_schema_v1.py --src benchmark_results_gemma3n/ckpt-20000 --ckpt ckpt-20000
    python generate_schema_v1.py --all   # process all gemma3n checkpoints
"""

import argparse
import json
import os
import re
import unicodedata
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path

import jiwer
import numpy as np

NORMALIZATION_VERSION = "v1"

# ── Normalization Pipeline ──────────────────────────────────────────────

ZERO_WIDTH = re.compile(r"[\u200b-\u200f\u2028-\u202f\ufeff\u00ad]")
PUNCT_COMMON = re.compile(
    r"[!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~"
    r"\u0964\u0965"
    r"\u2013\u2014\u2015"
    r"\u2018\u2019\u201c\u201d"
    r"\u2026\u00ab\u00bb\u0022\u0027"
    r"]"
)
DIGIT_GROUP_SEP = re.compile(r"(?<=\d)[,\s](?=\d)")

def norm_raw(text: str) -> str:
    """Tier 0 — NFC + trim."""
    return unicodedata.normalize("NFC", text).strip()

def norm_standard(text: str) -> str:
    """Tier 1 — NFKC + ZW strip + whitespace collapse + punct removal + case fold."""
    text = unicodedata.normalize("NFKC", text)
    text = ZERO_WIDTH.sub("", text)
    text = re.sub(r"\s+", " ", text).strip()
    text = text.replace("\u2018", "'").replace("\u2019", "'")
    text = text.replace("\u201c", '"').replace("\u201d", '"')
    text = text.replace("\u2013", "-").replace("\u2014", "-")
    text = text.replace("\u0965", "\u0964")
    text = PUNCT_COMMON.sub("", text)
    text = re.sub(r"\s+", " ", text).strip()
    text = text.lower()
    return text

def norm_numcanon(text: str) -> str:
    """Tier 2 — norm + digit grouping removal."""
    text = norm_standard(text)
    text = DIGIT_GROUP_SEP.sub("", text)
    return text

def norm_spaceless(text: str) -> str:
    """For MER — remove all spaces after standard norm."""
    return norm_standard(text).replace(" ", "")

# ── space_norm_wer (per-sample) ─────────────────────────────────────────

def compute_space_norm_wer_sample(ref_norm: str, hyp_norm: str) -> tuple:
    """Returns (error_words, total_words) for one sample."""
    ref_words = ref_norm.split()
    total_words = len(ref_words)
    if total_words == 0:
        return (0, 0)

    ref_ns = ref_norm.replace(" ", "")
    hyp_ns = hyp_norm.replace(" ", "")

    if ref_ns == hyp_ns:
        return (0, total_words)

    # Build char_to_word mapping
    char_to_word = []
    for wi, w in enumerate(ref_words):
        for _ in w:
            char_to_word.append(wi)

    # Levenshtein DP
    n, m = len(ref_ns), len(hyp_ns)
    dp = [[0] * (m + 1) for _ in range(n + 1)]
    for i in range(n + 1):
        dp[i][0] = i
    for j in range(m + 1):
        dp[0][j] = j
    for i in range(1, n + 1):
        for j in range(1, m + 1):
            if ref_ns[i - 1] == hyp_ns[j - 1]:
                dp[i][j] = dp[i - 1][j - 1]
            else:
                dp[i][j] = 1 + min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1])

    # Backtrack to find touched words
    touched = set()
    i, j = n, m
    while i > 0 or j > 0:
        if i > 0 and j > 0 and ref_ns[i - 1] == hyp_ns[j - 1]:
            i -= 1
            j -= 1
        elif i > 0 and j > 0 and dp[i][j] == dp[i - 1][j - 1] + 1:
            # substitution
            touched.add(char_to_word[i - 1])
            i -= 1
            j -= 1
        elif i > 0 and dp[i][j] == dp[i - 1][j] + 1:
            # deletion
            touched.add(char_to_word[i - 1])
            i -= 1
        elif j > 0 and dp[i][j] == dp[i][j - 1] + 1:
            # insertion — attribute to nearest ref word
            if i > 0:
                touched.add(char_to_word[i - 1])
            elif char_to_word:
                touched.add(char_to_word[0])
            j -= 1
        else:
            break

    return (len(touched), total_words)

# ── Per-sample WER/MER ──────────────────────────────────────────────────

def safe_wer(ref: str, hyp: str) -> float:
    if not ref.strip():
        return 0.0 if not hyp.strip() else 100.0
    try:
        return round(jiwer.wer(ref, hyp) * 100, 2)
    except Exception:
        return 100.0

def safe_cer(ref: str, hyp: str) -> float:
    if not ref.strip():
        return 0.0 if not hyp.strip() else 100.0
    try:
        return round(jiwer.cer(ref, hyp) * 100, 2)
    except Exception:
        return 100.0

# ── Flag detection ──────────────────────────────────────────────────────

def detect_flags(r_raw, h_raw, r_n, h_n, r_nc, h_nc, r_ns, h_ns, wer_norm_val) -> list:
    flags = []
    if r_raw == h_raw:
        flags.append("exact_match")
    if r_n == h_n:
        flags.append("exact_match_norm")
    if not h_raw.strip():
        flags.append("empty_hypothesis")
    if r_raw != h_raw and r_n == h_n:
        flags.append("punctuation_only_diff")
    if r_n != h_n and r_nc == h_nc:
        flags.append("numeric_mismatch")
    if wer_norm_val > 80:
        flags.append("high_wer")
    # spacing_error: norm differs but space-stripped matches
    if r_n != h_n and r_ns == h_ns:
        flags.append("spacing_error")
    return flags

# ── Error analysis ──────────────────────────────────────────────────────

def compute_error_ops(refs, hyps):
    subs, ins, dels = Counter(), Counter(), Counter()
    for ref, hyp in zip(refs, hyps):
        ref_words = ref.split()
        hyp_words = hyp.split()
        try:
            out = jiwer.process_words(ref, hyp)
            for chunk in out.alignments[0]:
                if chunk.type == "substitute":
                    for ri, hi in zip(range(chunk.ref_start_idx, chunk.ref_end_idx),
                                      range(chunk.hyp_start_idx, chunk.hyp_end_idx)):
                        if ri < len(ref_words) and hi < len(hyp_words):
                            subs[(ref_words[ri], hyp_words[hi])] += 1
                elif chunk.type == "insert":
                    for hi in range(chunk.hyp_start_idx, chunk.hyp_end_idx):
                        if hi < len(hyp_words):
                            ins[hyp_words[hi]] += 1
                elif chunk.type == "delete":
                    for ri in range(chunk.ref_start_idx, chunk.ref_end_idx):
                        if ri < len(ref_words):
                            dels[ref_words[ri]] += 1
        except Exception:
            continue
    return subs, ins, dels

# ── Main conversion ─────────────────────────────────────────────────────

def process_checkpoint(src_dir: str, dst_dir: str, ckpt_name: str):
    os.makedirs(dst_dir, exist_ok=True)

    with open(os.path.join(src_dir, "predictions.json")) as f:
        predictions = json.load(f)
    print(f"  Loaded {len(predictions)} predictions")

    # Load old meta
    with open(os.path.join(src_dir, "metrics.json")) as f:
        old_meta = json.load(f).get("__meta__", {})

    # ── Per-sample processing ───────────────────────────────────────
    by_lang = defaultdict(lambda: {
        "refs_raw": [], "hyps_raw": [],
        "refs_norm": [], "hyps_norm": [],
        "refs_nc": [], "hyps_nc": [],
        "refs_ns": [], "hyps_ns": [],
        "snwer_errors": [], "snwer_totals": [],
    })

    samples = []

    for p in predictions:
        lang = p["language"]
        ref, hyp = p["reference"], p.get("hypothesis", "")

        r_raw = norm_raw(ref)
        h_raw = norm_raw(hyp)
        r_n = norm_standard(ref)
        h_n = norm_standard(hyp)
        r_nc = norm_numcanon(ref)
        h_nc = norm_numcanon(hyp)
        r_ns = norm_spaceless(ref)
        h_ns = norm_spaceless(hyp)

        w_raw = safe_wer(r_raw, h_raw)
        w_norm = safe_wer(r_n, h_n)
        # MER = CER on space-stripped text (since it's a single "word")
        mer_val = safe_cer(r_ns, h_ns) if r_ns else (0.0 if not h_ns else 100.0)
        flags = detect_flags(r_raw, h_raw, r_n, h_n, r_nc, h_nc, r_ns, h_ns, w_norm)

        # space_norm_wer per sample
        sn_err, sn_tot = compute_space_norm_wer_sample(r_n, h_n)

        d = by_lang[lang]
        d["refs_raw"].append(r_raw); d["hyps_raw"].append(h_raw)
        d["refs_norm"].append(r_n); d["hyps_norm"].append(h_n)
        d["refs_nc"].append(r_nc); d["hyps_nc"].append(h_nc)
        d["refs_ns"].append(r_ns); d["hyps_ns"].append(h_ns)
        d["snwer_errors"].append(sn_err); d["snwer_totals"].append(sn_tot)

        samples.append({
            "id": p["id"],
            "language": lang,
            "reference": ref,
            "hypothesis": hyp,
            "ref_norm": r_n,
            "hyp_norm": h_n,
            "ref_numcanon": r_nc,
            "hyp_numcanon": h_nc,
            "ref_mer": r_ns,
            "hyp_mer": h_ns,
            "wer_raw": w_raw,
            "wer_norm": w_norm,
            "mer": mer_val,
            "flags": flags,
        })

    # ── Per-language metrics ────────────────────────────────────────
    metrics = {}
    agg = {"raw": ([], []), "norm": ([], []), "nc": ([], []), "ns": ([], []),
           "snwer_e": [], "snwer_t": []}

    for lang in sorted(by_lang.keys()):
        d = by_lang[lang]
        n = len(d["refs_raw"])

        w_raw = round(jiwer.wer(d["refs_raw"], d["hyps_raw"]) * 100, 2)
        w_norm = round(jiwer.wer(d["refs_norm"], d["hyps_norm"]) * 100, 2)
        w_nc = round(jiwer.wer(d["refs_nc"], d["hyps_nc"]) * 100, 2)
        c_norm = round(jiwer.cer(d["refs_norm"], d["hyps_norm"]) * 100, 2)

        # MER (corpus-level CER on space-stripped)
        all_ref_ns = "".join(d["refs_ns"])
        all_hyp_ns = "".join(d["hyps_ns"])
        mer_val = round(jiwer.cer(all_ref_ns, all_hyp_ns) * 100, 2) if all_ref_ns else 0.0

        # space_norm_wer (micro-average)
        sn_e = sum(d["snwer_errors"])
        sn_t = sum(d["snwer_totals"])
        snwer = round(sn_e / sn_t * 100, 2) if sn_t > 0 else 0.0

        empty_count = sum(1 for h in d["hyps_raw"] if not h.strip())

        metrics[lang] = {
            "n_samples": n,
            "wer_raw": w_raw,
            "wer_norm": w_norm,
            "wer_numcanon": w_nc,
            "space_norm_wer": snwer,
            "mer": mer_val,
            "cer_norm": c_norm,
            "empty_hypotheses": empty_count,
            "normalization_delta": {
                "raw_to_norm": round(w_raw - w_norm, 2),
                "norm_to_numcanon": round(w_norm - w_nc, 2),
                "norm_to_space_norm": round(w_norm - snwer, 2),
                "norm_to_mer": round(w_norm - mer_val, 2),
            },
        }

        agg["raw"][0].extend(d["refs_raw"]); agg["raw"][1].extend(d["hyps_raw"])
        agg["norm"][0].extend(d["refs_norm"]); agg["norm"][1].extend(d["hyps_norm"])
        agg["nc"][0].extend(d["refs_nc"]); agg["nc"][1].extend(d["hyps_nc"])
        agg["ns"][0].extend(d["refs_ns"]); agg["ns"][1].extend(d["hyps_ns"])
        agg["snwer_e"].extend(d["snwer_errors"]); agg["snwer_t"].extend(d["snwer_totals"])

    # ── Overall ─────────────────────────────────────────────────────
    all_ns_ref = "".join(agg["ns"][0])
    all_ns_hyp = "".join(agg["ns"][1])
    sn_e_all = sum(agg["snwer_e"])
    sn_t_all = sum(agg["snwer_t"])

    metrics["__overall__"] = {
        "n_samples": len(agg["raw"][0]),
        "wer_raw": round(jiwer.wer(agg["raw"][0], agg["raw"][1]) * 100, 2),
        "wer_norm": round(jiwer.wer(agg["norm"][0], agg["norm"][1]) * 100, 2),
        "wer_numcanon": round(jiwer.wer(agg["nc"][0], agg["nc"][1]) * 100, 2),
        "space_norm_wer": round(sn_e_all / sn_t_all * 100, 2) if sn_t_all else 0.0,
        "mer": round(jiwer.cer(all_ns_ref, all_ns_hyp) * 100, 2) if all_ns_ref else 0.0,
        "cer_norm": round(jiwer.cer(agg["norm"][0], agg["norm"][1]) * 100, 2),
    }

    # ── Macro average ───────────────────────────────────────────────
    lk = [k for k in metrics if not k.startswith("__")]
    metrics["__macro_avg__"] = {
        "n_languages": len(lk),
        "wer_raw": round(np.mean([metrics[k]["wer_raw"] for k in lk]), 2),
        "wer_norm": round(np.mean([metrics[k]["wer_norm"] for k in lk]), 2),
        "wer_numcanon": round(np.mean([metrics[k]["wer_numcanon"] for k in lk]), 2),
        "space_norm_wer": round(np.mean([metrics[k]["space_norm_wer"] for k in lk]), 2),
        "mer": round(np.mean([metrics[k]["mer"] for k in lk]), 2),
        "cer_norm": round(np.mean([metrics[k]["cer_norm"] for k in lk]), 2),
    }

    # ── Meta ────────────────────────────────────────────────────────
    metrics["__meta__"] = {
        "checkpoint": old_meta.get("checkpoint", ""),
        "checkpoint_name": ckpt_name,
        "model_id": "gemma3n-e2b",
        "model_type": old_meta.get("model_type", "gemma3n-E2B-asr"),
        "dataset": old_meta.get("dataset", "BayAreaBoys/indic-asr-benchmark-6k"),
        "batch_size": old_meta.get("batch_size", 128),
        "inference_time_sec": old_meta.get("inference_time_sec", 0),
        "total_audio_sec": old_meta.get("total_audio_sec", 0),
        "rtf": old_meta.get("rtf", 0),
        "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "gpu": "NVIDIA A100-SXM4-80GB",
        "framework": "transformers",
        "normalization_version": NORMALIZATION_VERSION,
        "jiwer_version": getattr(jiwer, "__version__", "unknown"),
    }

    # ── Write metrics.json ──────────────────────────────────────────
    with open(os.path.join(dst_dir, "metrics.json"), "w") as f:
        json.dump(metrics, f, indent=2, ensure_ascii=False)

    # ── Write sample_analysis.json ──────────────────────────────────
    with open(os.path.join(dst_dir, "sample_analysis.json"), "w") as f:
        json.dump(samples, f, indent=2, ensure_ascii=False)

    # ── Build error_analysis.json ───────────────────────────────────
    ea_by_lang = defaultdict(lambda: {"refs": [], "hyps": [], "ids": [], "wers": []})
    for s in samples:
        ea_by_lang[s["language"]]["refs"].append(s["ref_norm"])
        ea_by_lang[s["language"]]["hyps"].append(s["hyp_norm"])
        ea_by_lang[s["language"]]["ids"].append(s["id"])
        ea_by_lang[s["language"]]["wers"].append(s["wer_norm"])

    error_analysis = {}
    lang_avg_wers = {}

    for lang in sorted(ea_by_lang.keys()):
        d = ea_by_lang[lang]
        subs, ins, dels = compute_error_ops(d["refs"], d["hyps"])

        # Count error buckets from flags
        lang_samples = [s for s in samples if s["language"] == lang]
        num_mm = sum(1 for s in lang_samples if "numeric_mismatch" in s["flags"])
        punct_only = sum(1 for s in lang_samples if "punctuation_only_diff" in s["flags"])
        spacing_err = sum(1 for s in lang_samples if "spacing_error" in s["flags"])
        empty_hyp = sum(1 for s in lang_samples if "empty_hypothesis" in s["flags"])

        sorted_idx = np.argsort(d["wers"])
        worst = [d["ids"][i] for i in sorted_idx[-3:]][::-1]
        best = [d["ids"][i] for i in sorted_idx[:3]]
        num_samples = [s["id"] for s in lang_samples if "numeric_mismatch" in s["flags"]][:3]

        error_analysis[lang] = {
            "top_substitutions": [{"ref": r, "hyp": h, "count": c} for (r, h), c in subs.most_common(20)],
            "top_insertions": [{"word": w, "count": c} for w, c in ins.most_common(20)],
            "top_deletions": [{"word": w, "count": c} for w, c in dels.most_common(20)],
            "error_buckets": {
                "numeric_mismatch_count": num_mm,
                "punctuation_only_count": punct_only,
                "spacing_tokenization_count": spacing_err,
                "entity_mismatch_count": 0,
                "script_confusion_count": 0,
                "empty_hypothesis_count": empty_hyp,
            },
            "examples": {
                "worst_samples": worst,
                "best_samples": best,
                "numeric_mismatch_samples": num_samples,
            },
        }
        lang_avg_wers[lang] = np.mean(d["wers"])

    sorted_langs = sorted(lang_avg_wers, key=lang_avg_wers.get)
    ov = metrics["__overall__"]
    raw_norm_d = ov["wer_raw"] - ov["wer_norm"]
    norm_nc_d = ov["wer_norm"] - ov["wer_numcanon"]

    if raw_norm_d > 10:
        diag, primary = "formatting-limited", "formatting"
    elif norm_nc_d > 5:
        diag, primary = "numeric-limited", "numeric_verbalization"
    else:
        diag, primary = "recognition-limited", "recognition"

    error_analysis["__summary__"] = {
        "model_diagnosis": diag,
        "primary_error_source": primary,
        "numeric_verbalization_impact": "high" if norm_nc_d > 5 else "moderate" if norm_nc_d > 2 else "low",
        "formatting_impact": "high" if raw_norm_d > 10 else "moderate" if raw_norm_d > 5 else "low",
        "worst_languages": sorted_langs[-3:][::-1],
        "best_languages": sorted_langs[:3],
    }

    with open(os.path.join(dst_dir, "error_analysis.json"), "w") as f:
        json.dump(error_analysis, f, indent=2, ensure_ascii=False)

    # ── Print summary ───────────────────────────────────────────────
    print(f"  Overall: wer_raw={ov['wer_raw']} wer_norm={ov['wer_norm']} wer_nc={ov['wer_numcanon']} "
          f"snwer={ov['space_norm_wer']} mer={ov['mer']} cer_norm={ov['cer_norm']}")
    print(f"  Diagnosis: {diag}")
    return metrics

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--src", help="Source predictions dir")
    parser.add_argument("--ckpt", help="Checkpoint name")
    parser.add_argument("--all", action="store_true", help="Process all gemma3n checkpoints")
    args = parser.parse_args()

    dst_base = "/home/ubuntu/training/benchmark_outputs/gemma3n-e2b"

    if args.all:
        src_base = "/home/ubuntu/training/benchmark_results_gemma3n"
        for ckpt_dir in sorted(Path(src_base).iterdir()):
            if ckpt_dir.is_dir() and (ckpt_dir / "predictions.json").exists():
                ckpt = ckpt_dir.name
                print(f"\n{'='*60}\n  {ckpt}\n{'='*60}")
                process_checkpoint(str(ckpt_dir), f"{dst_base}/{ckpt}", ckpt)
    else:
        print(f"\n{'='*60}\n  {args.ckpt}\n{'='*60}")
        process_checkpoint(args.src, f"{dst_base}/{args.ckpt}", args.ckpt)

if __name__ == "__main__":
    main()


File created successfully at: /home/ubuntu/training/generate_schema_v1.py

#!/usr/bin/env python3
"""
Generate BENCHMARK_SCHEMA v1 outputs for gemma3n-e2b checkpoints.

Reads predictions.json → produces metrics.json, sample_analysis.json, error_analysis.json
with all 6 metric tiers: wer_raw, wer_norm, wer_numcanon, space_norm_wer, mer, cer_norm.

Usage:
    python generate_schema_v1.py --src benchmark_results_gemma3n/ckpt-20000 --ckpt ckpt-20000
    python generate_schema_v1.py --all   # process all gemma3n checkpoints
"""

import argparse
import json
import os
import re
import unicodedata
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path

import jiwer
import numpy as np

NORMALIZATION_VERSION = "v1"

# ── Normalization Pipeline ──────────────────────────────────────────────

ZERO_WIDTH = re.compile(r"[\u200b-\u200f\u2028-\u202f\ufeff\u00ad]")
PUNCT_COMMON = re.compile(
    r"[!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~"
    r"\u0964\u0965"
    r"\u2013\u2014\u2015"
    r"\u2018\u2019\u201c\u201d"
    r"\u2026\u00ab\u00bb\u0022\u0027"
    r"]"
)
DIGIT_GROUP_SEP = re.compile(r"(?<=\d)[,\s](?=\d)")

def norm_raw(text: str) -> str:
    """Tier 0 — NFC + trim."""
    return unicodedata.normalize("NFC", text).strip()

def norm_standard(text: str) -> str:
    """Tier 1 — NFKC + ZW strip + whitespace collapse + punct removal + case fold."""
    text = unicodedata.normalize("NFKC", text)
    text = ZERO_WIDTH.sub("", text)
    text = re.sub(r"\s+", " ", text).strip()
    text = text.replace("\u2018", "'").replace("\u2019", "'")
    text = text.replace("\u201c", '"').replace("\u201d", '"')
    text = text.replace("\u2013", "-").replace("\u2014", "-")
    text = text.replace("\u0965", "\u0964")
    text = PUNCT_COMMON.sub("", text)
    text = re.sub(r"\s+", " ", text).strip()
    text = text.lower()
    return text

def norm_numcanon(text: str) -> str:
    """Tier 2 — norm + digit grouping removal."""
    text = norm_standard(text)
    text = DIGIT_GROUP_SEP.sub("", text)
    return text

def norm_spaceless(text: str) -> str:
    """For MER — remove all spaces after standard norm."""
    return norm_standard(text).replace(" ", "")

# ── space_norm_wer (per-sample) ─────────────────────────────────────────

def compute_space_norm_wer_sample(ref_norm: str, hyp_norm: str) -> tuple:
    """Returns (error_words, total_words) for one sample."""
    ref_words = ref_norm.split()
    total_words = len(ref_words)
    if total_words == 0:
        return (0, 0)

    ref_ns = ref_norm.replace(" ", "")
    hyp_ns = hyp_norm.replace(" ", "")

    if ref_ns == hyp_ns:
        return (0, total_words)

    # Build char_to_word mapping
    char_to_word = []
    for wi, w in enumerate(ref_words):
        for _ in w:
            char_to_word.append(wi)

    # Levenshtein DP
    n, m = len(ref_ns), len(hyp_ns)
    dp = [[0] * (m + 1) for _ in range(n + 1)]
    for i in range(n + 1):
        dp[i][0] = i
    for j in range(m + 1):
        dp[0][j] = j
    for i in range(1, n + 1):
        for j in range(1, m + 1):
            if ref_ns[i - 1] == hyp_ns[j - 1]:
                dp[i][j] = dp[i - 1][j - 1]
            else:
                dp[i][j] = 1 + min(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1])

    # Backtrack to find touched words
    touched = set()
    i, j = n, m
    while i > 0 or j > 0:
        if i > 0 and j > 0 and ref_ns[i - 1] == hyp_ns[j - 1]:
            i -= 1
            j -= 1
        elif i > 0 and j > 0 and dp[i][j] == dp[i - 1][j - 1] + 1:
            # substitution
            touched.add(char_to_word[i - 1])
            i -= 1
            j -= 1
        elif i > 0 and dp[i][j] == dp[i - 1][j] + 1:
            # deletion
            touched.add(char_to_word[i - 1])
            i -= 1
        elif j > 0 and dp[i][j] == dp[i][j - 1] + 1:
            # insertion — attribute to nearest ref word
            if i > 0:
                touched.add(char_to_word[i - 1])
            elif char_to_word:
                touched.add(char_to_word[0])
            j -= 1
        else:
            break

    return (len(touched), total_words)

# ── Per-sample WER/MER ──────────────────────────────────────────────────

def safe_wer(ref: str, hyp: str) -> float:
    if not ref.strip():
        return 0.0 if not hyp.strip() else 100.0
    try:
        return round(jiwer.wer(ref, hyp) * 100, 2)
    except Exception:
        return 100.0

def safe_cer(ref: str, hyp: str) -> float:
    if not ref.strip():
        return 0.0 if not hyp.strip() else 100.0
    try:
        return round(jiwer.cer(ref, hyp) * 100, 2)
    except Exception:
        return 100.0

# ── Flag detection ──────────────────────────────────────────────────────

def detect_flags(r_raw, h_raw, r_n, h_n, r_nc, h_nc, r_ns, h_ns, wer_norm_val) -> list:
    flags = []
    if r_raw == h_raw:
        flags.append("exact_match")
    if r_n == h_n:
        flags.append("exact_match_norm")
    if not h_raw.strip():
        flags.append("empty_hypothesis")
    if r_raw != h_raw and r_n == h_n:
        flags.append("punctuation_only_diff")
    if r_n != h_n and r_nc == h_nc:
        flags.append("numeric_mismatch")
    if wer_norm_val > 80:
        flags.append("high_wer")
    # spacing_error: norm differs but space-stripped matches
    if r_n != h_n and r_ns == h_ns:
        flags.append("spacing_error")
    return flags

# ── Error analysis ──────────────────────────────────────────────────────

def compute_error_ops(refs, hyps):
    subs, ins, dels = Counter(), Counter(), Counter()
    for ref, hyp in zip(refs, hyps):
        ref_words = ref.split()
        hyp_words = hyp.split()
        try:
            out = jiwer.process_words(ref, hyp)
            for chunk in out.alignments[0]:
                if chunk.type == "substitute":
                    for ri, hi in zip(range(chunk.ref_start_idx, chunk.ref_end_idx),
                                      range(chunk.hyp_start_idx, chunk.hyp_end_idx)):
                        if ri < len(ref_words) and hi < len(hyp_words):
                            subs[(ref_words[ri], hyp_words[hi])] += 1
                elif chunk.type == "insert":
                    for hi in range(chunk.hyp_start_idx, chunk.hyp_end_idx):
                        if hi < len(hyp_words):
                            ins[hyp_words[hi]] += 1
                elif chunk.type == "delete":
                    for ri in range(chunk.ref_start_idx, chunk.ref_end_idx):
                        if ri < len(ref_words):
                            dels[ref_words[ri]] += 1
        except Exception:
            continue
    return subs, ins, dels

# ── Main conversion ─────────────────────────────────────────────────────

def process_checkpoint(src_dir: str, dst_dir: str, ckpt_name: str):
    os.makedirs(dst_dir, exist_ok=True)

    with open(os.path.join(src_dir, "predictions.json")) as f:
        predictions = json.load(f)
    print(f"  Loaded {len(predictions)} predictions")

    # Load old meta
    with open(os.path.join(src_dir, "metrics.json")) as f:
        old_meta = json.load(f).get("__meta__", {})

    # ── Per-sample processing ───────────────────────────────────────
    by_lang = defaultdict(lambda: {
        "refs_raw": [], "hyps_raw": [],
        "refs_norm": [], "hyps_norm": [],
        "refs_nc": [], "hyps_nc": [],
        "refs_ns": [], "hyps_ns": [],
        "snwer_errors": [], "snwer_totals": [],
    })

    samples = []

    for p in predictions:
        lang = p["language"]
        ref, hyp = p["reference"], p.get("hypothesis", "")

        r_raw = norm_raw(ref)
        h_raw = norm_raw(hyp)
        r_n = norm_standard(ref)
        h_n = norm_standard(hyp)
        r_nc = norm_numcanon(ref)
        h_nc = norm_numcanon(hyp)
        r_ns = norm_spaceless(ref)
        h_ns = norm_spaceless(hyp)

        w_raw = safe_wer(r_raw, h_raw)
        w_norm = safe_wer(r_n, h_n)
        # MER = CER on space-stripped text (since it's a single "word")
        mer_val = safe_cer(r_ns, h_ns) if r_ns else (0.0 if not h_ns else 100.0)
        flags = detect_flags(r_raw, h_raw, r_n, h_n, r_nc, h_nc, r_ns, h_ns, w_norm)

        # space_norm_wer per sample
        sn_err, sn_tot = compute_space_norm_wer_sample(r_n, h_n)

        d = by_lang[lang]
        d["refs_raw"].append(r_raw); d["hyps_raw"].append(h_raw)
        d["refs_norm"].append(r_n); d["hyps_norm"].append(h_n)
        d["refs_nc"].append(r_nc); d["hyps_nc"].append(h_nc)
        d["refs_ns"].append(r_ns); d["hyps_ns"].append(h_ns)
        d["snwer_errors"].append(sn_err); d["snwer_totals"].append(sn_tot)

        samples.append({
            "id": p["id"],
            "language": lang,
            "reference": ref,
            "hypothesis": hyp,
            "ref_norm": r_n,
            "hyp_norm": h_n,
            "ref_numcanon": r_nc,
            "hyp_numcanon": h_nc,
            "ref_mer": r_ns,
            "hyp_mer": h_ns,
            "wer_raw": w_raw,
            "wer_norm": w_norm,
            "mer": mer_val,
            "flags": flags,
        })

    # ── Per-language metrics ────────────────────────────────────────
    metrics = {}
    agg = {"raw": ([], []), "norm": ([], []), "nc": ([], []), "ns": ([], []),
           "snwer_e": [], "snwer_t": []}

    for lang in sorted(by_lang.keys()):
        d = by_lang[lang]
        n = len(d["refs_raw"])

        w_raw = round(jiwer.wer(d["refs_raw"], d["hyps_raw"]) * 100, 2)
        w_norm = round(jiwer.wer(d["refs_norm"], d["hyps_norm"]) * 100, 2)
        w_nc = round(jiwer.wer(d["refs_nc"], d["hyps_nc"]) * 100, 2)
        c_norm = round(jiwer.cer(d["refs_norm"], d["hyps_norm"]) * 100, 2)

        # MER (corpus-level CER on space-stripped)
        all_ref_ns = "".join(d["refs_ns"])
        all_hyp_ns = "".join(d["hyps_ns"])
        mer_val = round(jiwer.cer(all_ref_ns, all_hyp_ns) * 100, 2) if all_ref_ns else 0.0

        # space_norm_wer (micro-average)
        sn_e = sum(d["snwer_errors"])
        sn_t = sum(d["snwer_totals"])
        snwer = round(sn_e / sn_t * 100, 2) if sn_t > 0 else 0.0

        empty_count = sum(1 for h in d["hyps_raw"] if not h.strip())

        metrics[lang] = {
            "n_samples": n,
            "wer_raw": w_raw,
            "wer_norm": w_norm,
            "wer_numcanon": w_nc,
            "space_norm_wer": snwer,
            "mer": mer_val,
            "cer_norm": c_norm,
            "empty_hypotheses": empty_count,
            "normalization_delta": {
                "raw_to_norm": round(w_raw - w_norm, 2),
                "norm_to_numcanon": round(w_norm - w_nc, 2),
                "norm_to_space_norm": round(w_norm - snwer, 2),
                "norm_to_mer": round(w_norm - mer_val, 2),
            },
        }

        agg["raw"][0].extend(d["refs_raw"]); agg["raw"][1].extend(d["hyps_raw"])
        agg["norm"][0].extend(d["refs_norm"]); agg["norm"][1].extend(d["hyps_norm"])
        agg["nc"][0].extend(d["refs_nc"]); agg["nc"][1].extend(d["hyps_nc"])
        agg["ns"][0].extend(d["refs_ns"]); agg["ns"][1].extend(d["hyps_ns"])
        agg["snwer_e"].extend(d["snwer_errors"]); agg["snwer_t"].extend(d["snwer_totals"])

    # ── Overall ─────────────────────────────────────────────────────
    all_ns_ref = "".join(agg["ns"][0])
    all_ns_hyp = "".join(agg["ns"][1])
    sn_e_all = sum(agg["snwer_e"])
    sn_t_all = sum(agg["snwer_t"])

    metrics["__overall__"] = {
        "n_samples": len(agg["raw"][0]),
        "wer_raw": round(jiwer.wer(agg["raw"][0], agg["raw"][1]) * 100, 2),
        "wer_norm": round(jiwer.wer(agg["norm"][0], agg["norm"][1]) * 100, 2),
        "wer_numcanon": round(jiwer.wer(agg["nc"][0], agg["nc"][1]) * 100, 2),
        "space_norm_wer": round(sn_e_all / sn_t_all * 100, 2) if sn_t_all else 0.0,
        "mer": round(jiwer.cer(all_ns_ref, all_ns_hyp) * 100, 2) if all_ns_ref else 0.0,
        "cer_norm": round(jiwer.cer(agg["norm"][0], agg["norm"][1]) * 100, 2),
    }

    # ── Macro average ───────────────────────────────────────────────
    lk = [k for k in metrics if not k.startswith("__")]
    metrics["__macro_avg__"] = {
        "n_languages": len(lk),
        "wer_raw": round(np.mean([metrics[k]["wer_raw"] for k in lk]), 2),
        "wer_norm": round(np.mean([metrics[k]["wer_norm"] for k in lk]), 2),
        "wer_numcanon": round(np.mean([metrics[k]["wer_numcanon"] for k in lk]), 2),
        "space_norm_wer": round(np.mean([metrics[k]["space_norm_wer"] for k in lk]), 2),
        "mer": round(np.mean([metrics[k]["mer"] for k in lk]), 2),
        "cer_norm": round(np.mean([metrics[k]["cer_norm"] for k in lk]), 2),
    }

    # ── Meta ────────────────────────────────────────────────────────
    metrics["__meta__"] = {
        "checkpoint": old_meta.get("checkpoint", ""),
        "checkpoint_name": ckpt_name,
        "model_id": "gemma3n-e2b",
        "model_type": old_meta.get("model_type", "gemma3n-E2B-asr"),
        "dataset": old_meta.get("dataset", "BayAreaBoys/indic-asr-benchmark-6k"),
        "batch_size": old_meta.get("batch_size", 128),
        "inference_time_sec": old_meta.get("inference_time_sec", 0),
        "total_audio_sec": old_meta.get("total_audio_sec", 0),
        "rtf": old_meta.get("rtf", 0),
        "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
        "gpu": "NVIDIA A100-SXM4-80GB",
        "framework": "transformers",
        "normalization_version": NORMALIZATION_VERSION,
        "jiwer_version": getattr(jiwer, "__version__", "unknown"),
    }

    # ── Write metrics.json ──────────────────────────────────────────
    with open(os.path.join(dst_dir, "metrics.json"), "w") as f:
        json.dump(metrics, f, indent=2, ensure_ascii=False)

    # ── Write sample_analysis.json ──────────────────────────────────
    with open(os.path.join(dst_dir, "sample_analysis.json"), "w") as f:
        json.dump(samples, f, indent=2, ensure_ascii=False)

    # ── Build error_analysis.json ───────────────────────────────────
    ea_by_lang = defaultdict(lambda: {"refs": [], "hyps": [], "ids": [], "wers": []})
    for s in samples:
        ea_by_lang[s["language"]]["refs"].append(s["ref_norm"])
        ea_by_lang[s["language"]]["hyps"].append(s["hyp_norm"])
        ea_by_lang[s["language"]]["ids"].append(s["id"])
        ea_by_lang[s["language"]]["wers"].append(s["wer_norm"])

    error_analysis = {}
    lang_avg_wers = {}

    for lang in sorted(ea_by_lang.keys()):
        d = ea_by_lang[lang]
        subs, ins, dels = compute_error_ops(d["refs"], d["hyps"])

        # Count error buckets from flags
        lang_samples = [s for s in samples if s["language"] == lang]
        num_mm = sum(1 for s in lang_samples if "numeric_mismatch" in s["flags"])
        punct_only = sum(1 for s in lang_samples if "punctuation_only_diff" in s["flags"])
        spacing_err = sum(1 for s in lang_samples if "spacing_error" in s["flags"])
        empty_hyp = sum(1 for s in lang_samples if "empty_hypothesis" in s["flags"])

        sorted_idx = np.argsort(d["wers"])
        worst = [d["ids"][i] for i in sorted_idx[-3:]][::-1]
        best = [d["ids"][i] for i in sorted_idx[:3]]
        num_samples = [s["id"] for s in lang_samples if "numeric_mismatch" in s["flags"]][:3]

        error_analysis[lang] = {
            "top_substitutions": [{"ref": r, "hyp": h, "count": c} for (r, h), c in subs.most_common(20)],
            "top_insertions": [{"word": w, "count": c} for w, c in ins.most_common(20)],
            "top_deletions": [{"word": w, "count": c} for w, c in dels.most_common(20)],
            "error_buckets": {
                "numeric_mismatch_count": num_mm,
                "punctuation_only_count": punct_only,
                "spacing_tokenization_count": spacing_err,
                "entity_mismatch_count": 0,
                "script_confusion_count": 0,
                "empty_hypothesis_count": empty_hyp,
            },
            "examples": {
                "worst_samples": worst,
                "best_samples": best,
                "numeric_mismatch_samples": num_samples,
            },
        }
        lang_avg_wers[lang] = np.mean(d["wers"])

    sorted_langs = sorted(lang_avg_wers, key=lang_avg_wers.get)
    ov = metrics["__overall__"]
    raw_norm_d = ov["wer_raw"] - ov["wer_norm"]
    norm_nc_d = ov["wer_norm"] - ov["wer_numcanon"]

    if raw_norm_d > 10:
        diag, primary = "formatting-limited", "formatting"
    elif norm_nc_d > 5:
        diag, primary = "numeric-limited", "numeric_verbalization"
    else:
        diag, primary = "recognition-limited", "recognition"

    error_analysis["__summary__"] = {
        "model_diagnosis": diag,
        "primary_error_source": primary,
        "numeric_verbalization_impact": "high" if norm_nc_d > 5 else "moderate" if norm_nc_d > 2 else "low",
        "formatting_impact": "high" if raw_norm_d > 10 else "moderate" if raw_norm_d > 5 else "low",
        "worst_languages": sorted_langs[-3:][::-1],
        "best_languages": sorted_langs[:3],
    }

    with open(os.path.join(dst_dir, "error_analysis.json"), "w") as f:
        json.dump(error_analysis, f, indent=2, ensure_ascii=False)

    # ── Print summary ───────────────────────────────────────────────
    print(f"  Overall: wer_raw={ov['wer_raw']} wer_norm={ov['wer_norm']} wer_nc={ov['wer_numcanon']} "
          f"snwer={ov['space_norm_wer']} mer={ov['mer']} cer_norm={ov['cer_norm']}")
    print(f"  Diagnosis: {diag}")
    return metrics

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--src", help="Source predictions dir")
    parser.add_argument("--ckpt", help="Checkpoint name")
    parser.add_argument("--all", action="store_true", help="Process all gemma3n checkpoints")
    args = parser.parse_args()

    dst_base = "/home/ubuntu/training/benchmark_outputs/gemma3n-e2b"

    if args.all:
        src_base = "/home/ubuntu/training/benchmark_results_gemma3n"
        for ckpt_dir in sorted(Path(src_base).iterdir()):
            if ckpt_dir.is_dir() and (ckpt_dir / "predictions.json").exists():
                ckpt = ckpt_dir.name
                print(f"\n{'='*60}\n  {ckpt}\n{'='*60}")
                process_checkpoint(str(ckpt_dir), f"{dst_base}/{ckpt}", ckpt)
    else:
        print(f"\n{'='*60}\n  {args.ckpt}\n{'='*60}")
        process_checkpoint(args.src, f"{dst_base}/{args.ckpt}", args.ckpt)

if __name__ == "__main__":
    main()


Now let me check download progress.


Bash

du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


5.0G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
4.3G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


5.8G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
4.7G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


6.4G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
5.4G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


6.8G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
5.7G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


7.1G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
5.9G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


Downloading at ~500MB/min per checkpoint. Model is ~21GB each, so about 20 more minutes. Let me wait for them to finish.


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


7.4G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
6.1G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


7.5G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
6.4G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


7.7G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
6.6G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


7.8G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
6.8G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


7.8G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
7.0G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


7.9G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
7.2G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


7.9G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
7.5G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


8.1G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
7.8G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


8.2G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
8.1G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


8.4G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
8.5G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


8.7G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
8.5G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


9.0G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
8.6G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


9.1G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
8.8G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


9.3G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
9.0G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


~9GB each, ~12GB remaining per checkpoint. Let me check back in a few more rounds.


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


9.7G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
9.3G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


9.9G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
9.8G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


11G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
11G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


11G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
12G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


12G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
12G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


12G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
13G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


12G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
14G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


13G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
14G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


13G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
15G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


14G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
15G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


14G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
16G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


14G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
17G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


15G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
17G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


15G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
18G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


15G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
19G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


16G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
19G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


16G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
20G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check download progress


du -sh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1 && ls /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/*.safetensors 2>&1

Check if model file is complete


Error: Exit code 2
16G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/
20G	/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/
ls: cannot access '/home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/*.safetensors': No such file or directory


<task-notification>
<task-id>bcy99cwa3</task-id>
<tool-use-id>toolu_012r4DRTYktE4FA5pexBWKom</tool-use-id>
<output-file>/tmp/claude-1001/-home-ubuntu-training/5e207b1e-98a7-43a4-9947-9579680cf6bd/tasks/bcy99cwa3.output</output-file>
<status>completed</status>
<summary>Background command "Download ckpt-20000 model files" completed (exit code 0)</summary>
</task-notification>


ls -lh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-40000/ 2>&1

Check ckpt-40000 files


total 20G
-rw-rw-r-- 1 ubuntu ubuntu  613 Mar 29 06:43 chat_template.jinja
-rw-rw-r-- 1 ubuntu ubuntu 4.5K Mar 29 06:43 config.json
-rw-rw-r-- 1 ubuntu ubuntu  209 Mar 29 06:43 generation_config.json
-rw-rw-r-- 1 ubuntu ubuntu  20G Mar 29 07:40 model.safetensors.23be8a9f
-rw-rw-r-- 1 ubuntu ubuntu 2.3K Mar 29 06:43 processor_config.json
-rw-rw-r-- 1 ubuntu ubuntu 1004 Mar 29 06:43 tokenizer_config.json
-rw-rw-r-- 1 ubuntu ubuntu  32M Mar 29 06:43 tokenizer.json


ckpt-40000 still downloading (`.23be8a9f` is a temp suffix). ckpt-20000 at 16G — let me check its status.


ls -lh /home/ubuntu/training/checkpoints/gemma3n-e2b-ckpt-20000/ 2>&1

Check ckpt-20000 files


total 32M
-rw-rw-r-- 1 ubuntu ubuntu  613 Mar 28 06:31 chat_template.jinja
-rw-rw-r-- 1 ubuntu ubuntu 4.5K Mar 28 06:31 config.json
-rw-rw-r-- 1 ubuntu ubuntu  209 Mar 28 06:31 generation_config.json
-rw-rw-r-- 1 ubuntu ubuntu 2.3K Mar 28 06:31 processor_config.json
-rw-rw-r-- 1 ubuntu ubuntu 1004 Mar 28 06:31 tokenizer_config.json
-rw-rw-r-- 1 ubuntu ubuntu  32M Mar 28 06:31 tokenizer.json


ckpt-20000 is missing model.safetensors! The download may have failed. Let me check the background task output.


tail -10 /tmp/claude-1001/-home-ubuntu-training/5e207b1e-98a7-43a4-9947-9579680cf6bd/tasks/bcy99cwa3.output 2>&1

Check ckpt-20000 download output