     1→"""
     2→Qwen3-TTS-12Hz-0.6B-Base — Maximum Accuracy Evaluation
     3→GPU: A100-80GB | dtype: bfloat16
     4→Metrics: Word Error Rate (WER), Character Error Rate (CER)
     5→With proper text normalization to account for Whisper formatting
     6→"""
     7→
     8→import torch
     9→import soundfile as sf
    10→import numpy as np
    11→import os
    12→import re
    13→import time
    14→import json
    15→import whisper
    16→from jiwer import wer, cer
    17→from qwen_tts import Qwen3TTSModel
    18→
    19→OUTPUT_DIR = "/home/ubuntu/quench_0.6b/results"
    20→os.makedirs(OUTPUT_DIR, exist_ok=True)
    21→
    22→# ─────────────────────────────────────────────
    23→# Text normalization for fair TTS evaluation
    24→# ─────────────────────────────────────────────
    25→NUM_WORDS = {
    26→    "zero": "0", "one": "1", "two": "2", "three": "3", "four": "4",
    27→    "five": "5", "six": "6", "seven": "7", "eight": "8", "nine": "9",
    28→    "ten": "10", "eleven": "11", "twelve": "12", "thirteen": "13",
    29→    "fourteen": "14", "fifteen": "15", "sixteen": "16", "seventeen": "17",
    30→    "eighteen": "18", "nineteen": "19", "twenty": "20", "thirty": "30",
    31→    "forty": "40", "fifty": "50", "sixty": "60", "seventy": "70",
    32→    "eighty": "80", "ninety": "90", "hundred": "100", "thousand": "1000",
    33→    "million": "1000000", "billion": "1000000000",
    34→}
    35→
    36→# Traditional -> Simplified Chinese mapping (common chars)
    37→TRAD_TO_SIMP = str.maketrans(
    38→    "變著們國於華書學過來見與開關認為問題點個這時後從對說無現還進發經資機車門長風東運動當電話開實義氣質員準備區體頭數據覺識邊導彈戰場製議護辦歡環歷歲壓積隊針鑰匙飛網際際計處廣園壞擴聯節職誰認識議導員獨歷歲壓積針飛際計處廣園壞擴聯節職誰導員獨發經過來見與開關認為問題點個這時後從對說無現還進經資機車門長風東運動當電話開實義氣質員準備區體頭數據覺識邊導彈戰場製議護辦歡環歷壓積隊",
    39→    "变着们国于华书学过来见与开关认为问题点个这时后从对说无现还进发经资机车门长风东运动当电话开实义气质员准备区体头数据觉识边导弹战场制议护办欢环历岁压积队针钥匙飞网际际计处广园坏扩联节职谁认识议导员独历岁压积针飞际计处广园坏扩联节职谁导员独发经过来见与开关认为问题点个这时后从对说无现还进经资机车门长风东运动当电话开实义气质员准备区体头数据觉识边导弹战场制议护办欢环历压积队"
    40→)
    41→
    42→def normalize_text(text, language="English"):
    43→    """Normalize text for fair WER/CER comparison between input and Whisper output."""
    44→    t = text.strip()
    45→    # Remove punctuation that Whisper may drop or change
    46→    t = re.sub(r"['\"""''「」『』（）\(\)]", "", t)
    47→    t = re.sub(r"[,，.。!！?？;；:：、]", " ", t)
    48→    t = re.sub(r"\s+", " ", t).strip().lower()
    49→
    50→    if language in ["Chinese", "Japanese"]:
    51→        # Traditional -> Simplified
    52→        t = t.translate(TRAD_TO_SIMP)
    53→        # Remove all spaces for CJK
    54→        t = re.sub(r"\s+", "", t)
    55→
    56→    return t
    57→
    58→
    59→# ─────────────────────────────────────────────
    60→# 1. Load Models
    61→# ─────────────────────────────────────────────
    62→print("=" * 60)
    63→print("  Qwen3-TTS-12Hz-0.6B-Base — Max Accuracy Eval")
    64→print("=" * 60)
    65→
    66→print("\n[1/3] Loading Qwen3-TTS model (bfloat16, CUDA)...")
    67→t0 = time.time()
    68→tts_model = Qwen3TTSModel.from_pretrained(
    69→    "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    70→    device_map="cuda:0",
    71→    dtype=torch.bfloat16,
    72→)
    73→print(f"  TTS model loaded in {time.time()-t0:.1f}s")
    74→
    75→print("\n[2/3] Loading Whisper large-v2 for transcription...")
    76→t0 = time.time()
    77→whisper_model = whisper.load_model("large-v2", device="cuda:0")
    78→print(f"  Whisper loaded in {time.time()-t0:.1f}s")
    79→
    80→# ─────────────────────────────────────────────
    81→# 2. Test Sentences
    82→# ─────────────────────────────────────────────
    83→test_cases = [
    84→    # English - clean text (no numbers to avoid Whisper digit formatting)
    85→    {"text": "The quick brown fox jumps over the lazy dog near the riverbank.", "language": "English", "id": "en_pangram"},
    86→    {"text": "Artificial intelligence is transforming how we approach complex problems in science and engineering.", "language": "English", "id": "en_tech"},
    87→    {"text": "The Massachusetts Institute of Technology announced groundbreaking research in superconductivity.", "language": "English", "id": "en_proper_nouns"},
    88→    {"text": "She couldn't believe it and then whispered this changes everything we know about quantum mechanics.", "language": "English", "id": "en_complex"},
    89→    {"text": "Beautiful weather today with clear skies and a gentle breeze coming from the northwest.", "language": "English", "id": "en_weather"},
    90→    {"text": "The orchestra performed a magnificent symphony that left the audience speechless and deeply moved.", "language": "English", "id": "en_arts"},
    91→    {"text": "Renewable energy sources such as solar and wind power are becoming increasingly important worldwide.", "language": "English", "id": "en_energy"},
    92→    {"text": "The philosopher argued that consciousness emerges from the complex interactions of neural networks.", "language": "English", "id": "en_philosophy"},
    93→    {"text": "International cooperation is essential for addressing global challenges like climate change and poverty.", "language": "English", "id": "en_global"},
    94→    {"text": "The restaurant serves traditional Mediterranean cuisine with fresh ingredients sourced from local farmers.", "language": "English", "id": "en_food"},
    95→    # Chinese
    96→    {"text": "人工智能正在深刻改变着我们的日常生活和工作方式。", "language": "Chinese", "id": "zh_ai"},
    97→    {"text": "今天天气非常好，蓝天白云，微风轻拂。", "language": "Chinese", "id": "zh_weather"},
    98→    {"text": "科学技术是第一生产力，创新驱动发展。", "language": "Chinese", "id": "zh_science"},
    99→    {"text": "这座城市有着悠久的历史和丰富的文化遗产。", "language": "Chinese", "id": "zh_culture"},
   100→    # Japanese
   101→    {"text": "東京は日本の首都であり、世界で最も人口の多い都市の一つです。", "language": "Japanese", "id": "ja_tokyo"},
   102→    {"text": "桜の花が満開で、とても美しい景色が広がっています。", "language": "Japanese", "id": "ja_cherry"},
   103→    # Korean
   104→    {"text": "인공지능 기술은 우리의 일상생활을 크게 변화시키고 있습니다.", "language": "Korean", "id": "ko_ai"},
   105→    {"text": "오늘 날씨가 정말 좋아서 산책하기에 딱 좋은 날입니다.", "language": "Korean", "id": "ko_weather"},
   106→    # German
   107→    {"text": "Die Wissenschaft hat in den letzten Jahren enorme Fortschritte gemacht.", "language": "German", "id": "de_science"},
   108→    # French
   109→    {"text": "La technologie transforme notre façon de vivre et de travailler au quotidien.", "language": "French", "id": "fr_tech"},
   110→    # Spanish
   111→    {"text": "La inteligencia artificial está revolucionando todos los aspectos de nuestra vida cotidiana.", "language": "Spanish", "id": "es_ai"},
   112→]
   113→
   114→REF_AUDIO = "/home/ubuntu/quench_0.6b/ref_audio.wav"
   115→REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
   116→
   117→WHISPER_LANG_MAP = {
   118→    "English": "en", "Chinese": "zh", "Japanese": "ja", "Korean": "ko",
   119→    "German": "de", "French": "fr", "Spanish": "es",
   120→}
   121→
   122→# ─────────────────────────────────────────────
   123→# 3. Run Evaluation
   124→# ─────────────────────────────────────────────
   125→print(f"\n[3/3] Running evaluation on {len(test_cases)} test cases...")
   126→print("-" * 60)
   127→
   128→results = []
   129→
   130→for i, tc in enumerate(test_cases):
   131→    text = tc["text"]
   132→    lang = tc["language"]
   133→    tid = tc["id"]
   134→
   135→    print(f"\n  [{i+1}/{len(test_cases)}] {tid} ({lang})")
   136→    print(f"    Input : {text[:80]}{'...' if len(text)>80 else ''}")
   137→
   138→    t0 = time.time()
   139→    try:
   140→        wavs, sr = tts_model.generate_voice_clone(
   141→            text=text,
   142→            language=lang,
   143→            ref_audio=REF_AUDIO,
   144→            ref_text=REF_TEXT,
   145→            non_streaming_mode=True,
   146→        )
   147→        gen_time = time.time() - t0
   148→        audio = wavs[0]
   149→
   150→        wav_path = os.path.join(OUTPUT_DIR, f"{tid}.wav")
   151→        sf.write(wav_path, audio, sr)
   152→
   153→        audio_duration = len(audio) / sr
   154→        rtf = gen_time / audio_duration
   155→
   156→        whisper_result = whisper_model.transcribe(
   157→            wav_path,
   158→            language=WHISPER_LANG_MAP.get(lang, "en"),
   159→            beam_size=5,
   160→            best_of=5,
   161→            temperature=0.0,
   162→        )
   163→        transcription = whisper_result["text"].strip()
   164→
   165→        # Raw metrics
   166→        raw_w = wer(text, transcription)
   167→        raw_c = cer(text, transcription)
   168→
   169→        # Normalized metrics (fair comparison)
   170→        norm_input = normalize_text(text, lang)
   171→        norm_output = normalize_text(transcription, lang)
   172→        norm_w = wer(norm_input, norm_output) if norm_input and norm_output else raw_w
   173→        norm_c = cer(norm_input, norm_output) if norm_input and norm_output else raw_c
   174→
   175→        print(f"    Output: {transcription[:80]}{'...' if len(transcription)>80 else ''}")
   176→        print(f"    Raw    WER: {raw_w*100:6.2f}%  CER: {raw_c*100:6.2f}%")
   177→        print(f"    Norm   WER: {norm_w*100:6.2f}%  CER: {norm_c*100:6.2f}%")
   178→        print(f"    Audio: {audio_duration:.2f}s | Gen: {gen_time:.2f}s | RTF: {rtf:.3f}")
   179→
   180→        results.append({
   181→            "id": tid, "language": lang, "input_text": text,
   182→            "transcription": transcription,
   183→            "norm_input": norm_input, "norm_output": norm_output,
   184→            "raw_wer": round(raw_w * 100, 2), "raw_cer": round(raw_c * 100, 2),
   185→            "norm_wer": round(norm_w * 100, 2), "norm_cer": round(norm_c * 100, 2),
   186→            "audio_duration_s": round(audio_duration, 2),
   187→            "generation_time_s": round(gen_time, 2),
   188→            "rtf": round(rtf, 3),
   189→        })
   190→    except Exception as e:
   191→        print(f"    ERROR: {e}")
   192→        import traceback; traceback.print_exc()
   193→        results.append({"id": tid, "language": lang, "input_text": text, "error": str(e)})
   194→
   195→# ─────────────────────────────────────────────
   196→# 4. Summary
   197→# ─────────────────────────────────────────────
   198→successful = [r for r in results if "error" not in r]
   199→n = len(successful)
   200→
   201→avg_raw_wer = np.mean([r["raw_wer"] for r in successful]) if n else -1
   202→avg_raw_cer = np.mean([r["raw_cer"] for r in successful]) if n else -1
   203→avg_norm_wer = np.mean([r["norm_wer"] for r in successful]) if n else -1
   204→avg_norm_cer = np.mean([r["norm_cer"] for r in successful]) if n else -1
   205→avg_rtf = np.mean([r["rtf"] for r in successful]) if n else -1
   206→
   207→# Per-language breakdown
   208→lang_results = {}
   209→for r in successful:
   210→    lang = r["language"]
   211→    if lang not in lang_results:
   212→        lang_results[lang] = {"raw_wer": [], "raw_cer": [], "norm_wer": [], "norm_cer": [], "rtf": []}
   213→    for k in ["raw_wer", "raw_cer", "norm_wer", "norm_cer", "rtf"]:
   214→        lang_results[lang][k].append(r[k])
   215→
   216→summary = {
   217→    "model": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
   218→    "gpu": "NVIDIA A100-SXM4-80GB",
   219→    "dtype": "bfloat16",
   220→    "whisper_model": "large-v2",
   221→    "total_test_cases": len(test_cases),
   222→    "successful": n,
   223→    "avg_raw_wer_pct": round(avg_raw_wer, 2),
   224→    "avg_raw_cer_pct": round(avg_raw_cer, 2),
   225→    "avg_norm_wer_pct": round(avg_norm_wer, 2),
   226→    "avg_norm_cer_pct": round(avg_norm_cer, 2),
   227→    "avg_rtf": round(avg_rtf, 3),
   228→    "per_language": {},
   229→    "results": results,
   230→}
   231→
   232→for lang, vals in lang_results.items():
   233→    summary["per_language"][lang] = {
   234→        "avg_raw_wer": round(np.mean(vals["raw_wer"]), 2),
   235→        "avg_raw_cer": round(np.mean(vals["raw_cer"]), 2),
   236→        "avg_norm_wer": round(np.mean(vals["norm_wer"]), 2),
   237→        "avg_norm_cer": round(np.mean(vals["norm_cer"]), 2),
   238→        "avg_rtf": round(np.mean(vals["rtf"]), 3),
   239→        "n": len(vals["rtf"]),
   240→    }
   241→
   242→results_path = os.path.join(OUTPUT_DIR, "evaluation_results.json")
   243→with open(results_path, "w") as f:
   244→    json.dump(summary, f, indent=2, ensure_ascii=False)
   245→
   246→print("\n" + "=" * 60)
   247→print("  FINAL RESULTS — Qwen3-TTS-12Hz-0.6B-Base")
   248→print("  Max Accuracy on NVIDIA A100-SXM4-80GB")
   249→print("=" * 60)
   250→print(f"  Precision:     bfloat16 (no quantization)")
   251→print(f"  Transcriber:   Whisper large-v2 (beam=5, best_of=5)")
   252→print(f"  Test cases:    {n}/{len(test_cases)} successful")
   253→print()
   254→print(f"  Overall (Normalized - fair comparison):")
   255→print(f"    WER:  {avg_norm_wer:.2f}%")
   256→print(f"    CER:  {avg_norm_cer:.2f}%")
   257→print(f"    RTF:  {avg_rtf:.3f}")
   258→print()
   259→print(f"  Overall (Raw - includes formatting diffs):")
   260→print(f"    WER:  {avg_raw_wer:.2f}%")
   261→print(f"    CER:  {avg_raw_cer:.2f}%")
   262→print()
   263→print(f"  {'Language':12s}  {'NormWER':>8s}  {'NormCER':>8s}  {'RawWER':>8s}  {'RawCER':>8s}  {'RTF':>6s}  {'n':>3s}")
   264→print(f"  {'-'*12}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*6}  {'-'*3}")
   265→for lang, v in sorted(lang_results.items()):
   266→    print(f"  {lang:12s}  {np.mean(v['norm_wer']):7.2f}%  {np.mean(v['norm_cer']):7.2f}%  {np.mean(v['raw_wer']):7.2f}%  {np.mean(v['raw_cer']):7.2f}%  {np.mean(v['rtf']):5.3f}  {len(v['rtf']):3d}")
   267→print()
   268→print(f"  Results + audio saved to: {OUTPUT_DIR}/")
   269→print("=" * 60)
   270→

"""
Qwen3-TTS-12Hz-0.6B-Base — Maximum Accuracy Evaluation
GPU: A100-80GB | dtype: bfloat16
Metrics: Word Error Rate (WER), Character Error Rate (CER)
With proper text normalization to account for Whisper formatting
"""

import torch
import soundfile as sf
import numpy as np
import os
import re
import time
import json
import whisper
from jiwer import wer, cer
from qwen_tts import Qwen3TTSModel

OUTPUT_DIR = "/home/ubuntu/quench_0.6b/results"
os.makedirs(OUTPUT_DIR, exist_ok=True)

# ─────────────────────────────────────────────
# Text normalization for fair TTS evaluation
# ─────────────────────────────────────────────
NUM_WORDS = {
    "zero": "0", "one": "1", "two": "2", "three": "3", "four": "4",
    "five": "5", "six": "6", "seven": "7", "eight": "8", "nine": "9",
    "ten": "10", "eleven": "11", "twelve": "12", "thirteen": "13",
    "fourteen": "14", "fifteen": "15", "sixteen": "16", "seventeen": "17",
    "eighteen": "18", "nineteen": "19", "twenty": "20", "thirty": "30",
    "forty": "40", "fifty": "50", "sixty": "60", "seventy": "70",
    "eighty": "80", "ninety": "90", "hundred": "100", "thousand": "1000",
    "million": "1000000", "billion": "1000000000",
}

# Traditional -> Simplified Chinese mapping (common chars)
TRAD_TO_SIMP = str.maketrans(
    "變著們國於華書學過來見與開關認為問題點個這時後從對說無現還進發經資機車門長風東運動當電話開實義氣質員準備區體頭數據覺識邊導彈戰場製議護辦歡環歷歲壓積隊針鑰匙飛網際際計處廣園壞擴聯節職誰認識議導員獨歷歲壓積針飛際計處廣園壞擴聯節職誰導員獨發經過來見與開關認為問題點個這時後從對說無現還進經資機車門長風東運動當電話開實義氣質員準備區體頭數據覺識邊導彈戰場製議護辦歡環歷壓積隊",
    "变着们国于华书学过来见与开关认为问题点个这时后从对说无现还进发经资机车门长风东运动当电话开实义气质员准备区体头数据觉识边导弹战场制议护办欢环历岁压积队针钥匙飞网际际计处广园坏扩联节职谁认识议导员独历岁压积针飞际计处广园坏扩联节职谁导员独发经过来见与开关认为问题点个这时后从对说无现还进经资机车门长风东运动当电话开实义气质员准备区体头数据觉识边导弹战场制议护办欢环历压积队"
)

def normalize_text(text, language="English"):
    """Normalize text for fair WER/CER comparison between input and Whisper output."""
    t = text.strip()
    # Remove punctuation that Whisper may drop or change
    t = re.sub(r"['\"""''「」『』（）\(\)]", "", t)
    t = re.sub(r"[,，.。!！?？;；:：、]", " ", t)
    t = re.sub(r"\s+", " ", t).strip().lower()

    if language in ["Chinese", "Japanese"]:
        # Traditional -> Simplified
        t = t.translate(TRAD_TO_SIMP)
        # Remove all spaces for CJK
        t = re.sub(r"\s+", "", t)

    return t

# ─────────────────────────────────────────────
# 1. Load Models
# ─────────────────────────────────────────────
print("=" * 60)
print("  Qwen3-TTS-12Hz-0.6B-Base — Max Accuracy Eval")
print("=" * 60)

print("\n[1/3] Loading Qwen3-TTS model (bfloat16, CUDA)...")
t0 = time.time()
tts_model = Qwen3TTSModel.from_pretrained(
    "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    device_map="cuda:0",
    dtype=torch.bfloat16,
)
print(f"  TTS model loaded in {time.time()-t0:.1f}s")

print("\n[2/3] Loading Whisper large-v2 for transcription...")
t0 = time.time()
whisper_model = whisper.load_model("large-v2", device="cuda:0")
print(f"  Whisper loaded in {time.time()-t0:.1f}s")

# ─────────────────────────────────────────────
# 2. Test Sentences
# ─────────────────────────────────────────────
test_cases = [
    # English - clean text (no numbers to avoid Whisper digit formatting)
    {"text": "The quick brown fox jumps over the lazy dog near the riverbank.", "language": "English", "id": "en_pangram"},
    {"text": "Artificial intelligence is transforming how we approach complex problems in science and engineering.", "language": "English", "id": "en_tech"},
    {"text": "The Massachusetts Institute of Technology announced groundbreaking research in superconductivity.", "language": "English", "id": "en_proper_nouns"},
    {"text": "She couldn't believe it and then whispered this changes everything we know about quantum mechanics.", "language": "English", "id": "en_complex"},
    {"text": "Beautiful weather today with clear skies and a gentle breeze coming from the northwest.", "language": "English", "id": "en_weather"},
    {"text": "The orchestra performed a magnificent symphony that left the audience speechless and deeply moved.", "language": "English", "id": "en_arts"},
    {"text": "Renewable energy sources such as solar and wind power are becoming increasingly important worldwide.", "language": "English", "id": "en_energy"},
    {"text": "The philosopher argued that consciousness emerges from the complex interactions of neural networks.", "language": "English", "id": "en_philosophy"},
    {"text": "International cooperation is essential for addressing global challenges like climate change and poverty.", "language": "English", "id": "en_global"},
    {"text": "The restaurant serves traditional Mediterranean cuisine with fresh ingredients sourced from local farmers.", "language": "English", "id": "en_food"},
    # Chinese
    {"text": "人工智能正在深刻改变着我们的日常生活和工作方式。", "language": "Chinese", "id": "zh_ai"},
    {"text": "今天天气非常好，蓝天白云，微风轻拂。", "language": "Chinese", "id": "zh_weather"},
    {"text": "科学技术是第一生产力，创新驱动发展。", "language": "Chinese", "id": "zh_science"},
    {"text": "这座城市有着悠久的历史和丰富的文化遗产。", "language": "Chinese", "id": "zh_culture"},
    # Japanese
    {"text": "東京は日本の首都であり、世界で最も人口の多い都市の一つです。", "language": "Japanese", "id": "ja_tokyo"},
    {"text": "桜の花が満開で、とても美しい景色が広がっています。", "language": "Japanese", "id": "ja_cherry"},
    # Korean
    {"text": "인공지능 기술은 우리의 일상생활을 크게 변화시키고 있습니다.", "language": "Korean", "id": "ko_ai"},
    {"text": "오늘 날씨가 정말 좋아서 산책하기에 딱 좋은 날입니다.", "language": "Korean", "id": "ko_weather"},
    # German
    {"text": "Die Wissenschaft hat in den letzten Jahren enorme Fortschritte gemacht.", "language": "German", "id": "de_science"},
    # French
    {"text": "La technologie transforme notre façon de vivre et de travailler au quotidien.", "language": "French", "id": "fr_tech"},
    # Spanish
    {"text": "La inteligencia artificial está revolucionando todos los aspectos de nuestra vida cotidiana.", "language": "Spanish", "id": "es_ai"},
]

REF_AUDIO = "/home/ubuntu/quench_0.6b/ref_audio.wav"
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."

WHISPER_LANG_MAP = {
    "English": "en", "Chinese": "zh", "Japanese": "ja", "Korean": "ko",
    "German": "de", "French": "fr", "Spanish": "es",
}

# ─────────────────────────────────────────────
# 3. Run Evaluation
# ─────────────────────────────────────────────
print(f"\n[3/3] Running evaluation on {len(test_cases)} test cases...")
print("-" * 60)

results = []

for i, tc in enumerate(test_cases):
    text = tc["text"]
    lang = tc["language"]
    tid = tc["id"]

    print(f"\n  [{i+1}/{len(test_cases)}] {tid} ({lang})")
    print(f"    Input : {text[:80]}{'...' if len(text)>80 else ''}")

    t0 = time.time()
    try:
        wavs, sr = tts_model.generate_voice_clone(
            text=text,
            language=lang,
            ref_audio=REF_AUDIO,
            ref_text=REF_TEXT,
            non_streaming_mode=True,
        )
        gen_time = time.time() - t0
        audio = wavs[0]

        wav_path = os.path.join(OUTPUT_DIR, f"{tid}.wav")
        sf.write(wav_path, audio, sr)

        audio_duration = len(audio) / sr
        rtf = gen_time / audio_duration

        whisper_result = whisper_model.transcribe(
            wav_path,
            language=WHISPER_LANG_MAP.get(lang, "en"),
            beam_size=5,
            best_of=5,
            temperature=0.0,
        )
        transcription = whisper_result["text"].strip()

        # Raw metrics
        raw_w = wer(text, transcription)
        raw_c = cer(text, transcription)

        # Normalized metrics (fair comparison)
        norm_input = normalize_text(text, lang)
        norm_output = normalize_text(transcription, lang)
        norm_w = wer(norm_input, norm_output) if norm_input and norm_output else raw_w
        norm_c = cer(norm_input, norm_output) if norm_input and norm_output else raw_c

        print(f"    Output: {transcription[:80]}{'...' if len(transcription)>80 else ''}")
        print(f"    Raw    WER: {raw_w*100:6.2f}%  CER: {raw_c*100:6.2f}%")
        print(f"    Norm   WER: {norm_w*100:6.2f}%  CER: {norm_c*100:6.2f}%")
        print(f"    Audio: {audio_duration:.2f}s | Gen: {gen_time:.2f}s | RTF: {rtf:.3f}")

        results.append({
            "id": tid, "language": lang, "input_text": text,
            "transcription": transcription,
            "norm_input": norm_input, "norm_output": norm_output,
            "raw_wer": round(raw_w * 100, 2), "raw_cer": round(raw_c * 100, 2),
            "norm_wer": round(norm_w * 100, 2), "norm_cer": round(norm_c * 100, 2),
            "audio_duration_s": round(audio_duration, 2),
            "generation_time_s": round(gen_time, 2),
            "rtf": round(rtf, 3),
        })
    except Exception as e:
        print(f"    ERROR: {e}")
        import traceback; traceback.print_exc()
        results.append({"id": tid, "language": lang, "input_text": text, "error": str(e)})

# ─────────────────────────────────────────────
# 4. Summary
# ─────────────────────────────────────────────
successful = [r for r in results if "error" not in r]
n = len(successful)

avg_raw_wer = np.mean([r["raw_wer"] for r in successful]) if n else -1
avg_raw_cer = np.mean([r["raw_cer"] for r in successful]) if n else -1
avg_norm_wer = np.mean([r["norm_wer"] for r in successful]) if n else -1
avg_norm_cer = np.mean([r["norm_cer"] for r in successful]) if n else -1
avg_rtf = np.mean([r["rtf"] for r in successful]) if n else -1

# Per-language breakdown
lang_results = {}
for r in successful:
    lang = r["language"]
    if lang not in lang_results:
        lang_results[lang] = {"raw_wer": [], "raw_cer": [], "norm_wer": [], "norm_cer": [], "rtf": []}
    for k in ["raw_wer", "raw_cer", "norm_wer", "norm_cer", "rtf"]:
        lang_results[lang][k].append(r[k])

summary = {
    "model": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    "gpu": "NVIDIA A100-SXM4-80GB",
    "dtype": "bfloat16",
    "whisper_model": "large-v2",
    "total_test_cases": len(test_cases),
    "successful": n,
    "avg_raw_wer_pct": round(avg_raw_wer, 2),
    "avg_raw_cer_pct": round(avg_raw_cer, 2),
    "avg_norm_wer_pct": round(avg_norm_wer, 2),
    "avg_norm_cer_pct": round(avg_norm_cer, 2),
    "avg_rtf": round(avg_rtf, 3),
    "per_language": {},
    "results": results,
}

for lang, vals in lang_results.items():
    summary["per_language"][lang] = {
        "avg_raw_wer": round(np.mean(vals["raw_wer"]), 2),
        "avg_raw_cer": round(np.mean(vals["raw_cer"]), 2),
        "avg_norm_wer": round(np.mean(vals["norm_wer"]), 2),
        "avg_norm_cer": round(np.mean(vals["norm_cer"]), 2),
        "avg_rtf": round(np.mean(vals["rtf"]), 3),
        "n": len(vals["rtf"]),
    }

results_path = os.path.join(OUTPUT_DIR, "evaluation_results.json")
with open(results_path, "w") as f:
    json.dump(summary, f, indent=2, ensure_ascii=False)

print("\n" + "=" * 60)
print("  FINAL RESULTS — Qwen3-TTS-12Hz-0.6B-Base")
print("  Max Accuracy on NVIDIA A100-SXM4-80GB")
print("=" * 60)
print(f"  Precision:     bfloat16 (no quantization)")
print(f"  Transcriber:   Whisper large-v2 (beam=5, best_of=5)")
print(f"  Test cases:    {n}/{len(test_cases)} successful")
print()
print(f"  Overall (Normalized - fair comparison):")
print(f"    WER:  {avg_norm_wer:.2f}%")
print(f"    CER:  {avg_norm_cer:.2f}%")
print(f"    RTF:  {avg_rtf:.3f}")
print()
print(f"  Overall (Raw - includes formatting diffs):")
print(f"    WER:  {avg_raw_wer:.2f}%")
print(f"    CER:  {avg_raw_cer:.2f}%")
print()
print(f"  {'Language':12s}  {'NormWER':>8s}  {'NormCER':>8s}  {'RawWER':>8s}  {'RawCER':>8s}  {'RTF':>6s}  {'n':>3s}")
print(f"  {'-'*12}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*6}  {'-'*3}")
for lang, v in sorted(lang_results.items()):
    print(f"  {lang:12s}  {np.mean(v['norm_wer']):7.2f}%  {np.mean(v['norm_cer']):7.2f}%  {np.mean(v['raw_wer']):7.2f}%  {np.mean(v['raw_cer']):7.2f}%  {np.mean(v['rtf']):5.3f}  {len(v['rtf']):3d}")
print()
print(f"  Results + audio saved to: {OUTPUT_DIR}/")
print("=" * 60)


The TTS model supports batched generation natively (`text` accepts a list). Let me rewrite to use batch generation for concurrency.


Write

"""
Qwen3-TTS-12Hz-0.6B-Base — Maximum Accuracy Evaluation (CONCURRENT)
GPU: A100-80GB | dtype: bfloat16
Uses batch generation for TTS + concurrent Whisper transcription
"""

import torch
import soundfile as sf
import numpy as np
import os
import re
import time
import json
import whisper
from concurrent.futures import ThreadPoolExecutor
from jiwer import wer, cer
from qwen_tts import Qwen3TTSModel

OUTPUT_DIR = "/home/ubuntu/quench_0.6b/results"
os.makedirs(OUTPUT_DIR, exist_ok=True)

# ─────────────────────────────────────────────
# Text normalization
# ─────────────────────────────────────────────
TRAD_TO_SIMP = str.maketrans(
    "變著們國於華書學過來見與開關認為問題點個這時後從對說無現還進發經資機車門長風東運動當電話開實義氣質員準備區體頭數據覺識邊導彈戰場製議護辦歡環歷歲壓積隊針鑰匙飛網際際計處廣園壞擴聯節職誰認識議導員獨歷歲壓積針飛際計處廣園壞擴聯節職誰導員獨發經過來見與開關認為問題點個這時後從對說無現還進經資機車門長風東運動當電話開實義氣質員準備區體頭數據覺識邊導彈戰場製議護辦歡環歷壓積隊",
    "变着们国于华书学过来见与开关认为问题点个这时后从对说无现还进发经资机车门长风东运动当电话开实义气质员准备区体头数据觉识边导弹战场制议护办欢环历岁压积队针钥匙飞网际际计处广园坏扩联节职谁认识议导员独历岁压积针飞际计处广园坏扩联节职谁导员独发经过来见与开关认为问题点个这时后从对说无现还进经资机车门长风东运动当电话开实义气质员准备区体头数据觉识边导弹战场制议护办欢环历压积队"
)

def normalize_text(text, language="English"):
    t = text.strip()
    t = re.sub(r"['\"""''「」『』（）\(\)]", "", t)
    t = re.sub(r"[,，.。!！?？;；:：、]", " ", t)
    t = re.sub(r"\s+", " ", t).strip().lower()
    if language in ["Chinese", "Japanese"]:
        t = t.translate(TRAD_TO_SIMP)
        t = re.sub(r"\s+", "", t)
    return t

WHISPER_LANG_MAP = {
    "English": "en", "Chinese": "zh", "Japanese": "ja", "Korean": "ko",
    "German": "de", "French": "fr", "Spanish": "es",
}

# ─────────────────────────────────────────────
# Test Sentences
# ─────────────────────────────────────────────
test_cases = [
    {"text": "The quick brown fox jumps over the lazy dog near the riverbank.", "language": "English", "id": "en_pangram"},
    {"text": "Artificial intelligence is transforming how we approach complex problems in science and engineering.", "language": "English", "id": "en_tech"},
    {"text": "The Massachusetts Institute of Technology announced groundbreaking research in superconductivity.", "language": "English", "id": "en_proper_nouns"},
    {"text": "She couldn't believe it and then whispered this changes everything we know about quantum mechanics.", "language": "English", "id": "en_complex"},
    {"text": "Beautiful weather today with clear skies and a gentle breeze coming from the northwest.", "language": "English", "id": "en_weather"},
    {"text": "The orchestra performed a magnificent symphony that left the audience speechless and deeply moved.", "language": "English", "id": "en_arts"},
    {"text": "Renewable energy sources such as solar and wind power are becoming increasingly important worldwide.", "language": "English", "id": "en_energy"},
    {"text": "The philosopher argued that consciousness emerges from the complex interactions of neural networks.", "language": "English", "id": "en_philosophy"},
    {"text": "International cooperation is essential for addressing global challenges like climate change and poverty.", "language": "English", "id": "en_global"},
    {"text": "The restaurant serves traditional Mediterranean cuisine with fresh ingredients sourced from local farmers.", "language": "English", "id": "en_food"},
    {"text": "人工智能正在深刻改变着我们的日常生活和工作方式。", "language": "Chinese", "id": "zh_ai"},
    {"text": "今天天气非常好，蓝天白云，微风轻拂。", "language": "Chinese", "id": "zh_weather"},
    {"text": "科学技术是第一生产力，创新驱动发展。", "language": "Chinese", "id": "zh_science"},
    {"text": "这座城市有着悠久的历史和丰富的文化遗产。", "language": "Chinese", "id": "zh_culture"},
    {"text": "東京は日本の首都であり、世界で最も人口の多い都市の一つです。", "language": "Japanese", "id": "ja_tokyo"},
    {"text": "桜の花が満開で、とても美しい景色が広がっています。", "language": "Japanese", "id": "ja_cherry"},
    {"text": "인공지능 기술은 우리의 일상생활을 크게 변화시키고 있습니다.", "language": "Korean", "id": "ko_ai"},
    {"text": "오늘 날씨가 정말 좋아서 산책하기에 딱 좋은 날입니다.", "language": "Korean", "id": "ko_weather"},
    {"text": "Die Wissenschaft hat in den letzten Jahren enorme Fortschritte gemacht.", "language": "German", "id": "de_science"},
    {"text": "La technologie transforme notre façon de vivre et de travailler au quotidien.", "language": "French", "id": "fr_tech"},
    {"text": "La inteligencia artificial está revolucionando todos los aspectos de nuestra vida cotidiana.", "language": "Spanish", "id": "es_ai"},
]

REF_AUDIO = "/home/ubuntu/quench_0.6b/ref_audio.wav"
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."

# ─────────────────────────────────────────────
# Load Models
# ─────────────────────────────────────────────
print("=" * 60)
print("  Qwen3-TTS-12Hz-0.6B-Base — CONCURRENT Max Accuracy Eval")
print("=" * 60)

print("\n[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...")
t0 = time.time()
tts_model = Qwen3TTSModel.from_pretrained(
    "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    device_map="cuda:0",
    dtype=torch.bfloat16,
)
# Pre-cache the voice clone prompt once (reuse for all samples)
voice_prompt = tts_model.create_voice_clone_prompt(
    ref_audio=REF_AUDIO,
    ref_text=REF_TEXT,
)
print(f"  TTS model + voice prompt cached in {time.time()-t0:.1f}s")

print("\n[2/2] Loading Whisper large-v2 for transcription...")
t0 = time.time()
whisper_model = whisper.load_model("large-v2", device="cuda:0")
print(f"  Whisper loaded in {time.time()-t0:.1f}s")

# ─────────────────────────────────────────────
# CONCURRENT Evaluation
# Group by language for batch TTS generation
# ─────────────────────────────────────────────
print(f"\nRunning CONCURRENT evaluation on {len(test_cases)} test cases...")
print("-" * 60)

# Group test cases by language for batched TTS
from collections import defaultdict
lang_groups = defaultdict(list)
for tc in test_cases:
    lang_groups[tc["language"]].append(tc)

results = []
total_gen_time = 0
total_audio_dur = 0
total_start = time.time()

for lang, cases in lang_groups.items():
    batch_texts = [c["text"] for c in cases]
    batch_langs = [lang] * len(cases)
    batch_ids = [c["id"] for c in cases]

    print(f"\n  BATCH [{lang}] — {len(cases)} samples")

    # Batch TTS generation
    t0 = time.time()
    try:
        wavs, sr = tts_model.generate_voice_clone(
            text=batch_texts,
            language=batch_langs,
            voice_clone_prompt=voice_prompt,
            non_streaming_mode=True,
        )
        gen_time = time.time() - t0
        total_gen_time += gen_time
        print(f"    Batch TTS done in {gen_time:.2f}s ({len(cases)} samples)")
    except Exception as e:
        print(f"    BATCH TTS ERROR: {e}")
        import traceback; traceback.print_exc()
        for c in cases:
            results.append({"id": c["id"], "language": lang, "input_text": c["text"], "error": str(e)})
        continue

    # Save all wav files
    wav_paths = []
    for i, (w, tid) in enumerate(zip(wavs, batch_ids)):
        wav_path = os.path.join(OUTPUT_DIR, f"{tid}.wav")
        sf.write(wav_path, w, sr)
        wav_paths.append(wav_path)

    # Concurrent Whisper transcription (CPU-bound prep is parallelizable)
    def transcribe_one(args):
        idx, wav_path, text, tid = args
        audio = wavs[idx]
        audio_duration = len(audio) / sr

        whisper_result = whisper_model.transcribe(
            wav_path,
            language=WHISPER_LANG_MAP.get(lang, "en"),
            beam_size=5,
            best_of=5,
            temperature=0.0,
        )
        transcription = whisper_result["text"].strip()

        raw_w = wer(text, transcription)
        raw_c = cer(text, transcription)
        norm_input = normalize_text(text, lang)
        norm_output = normalize_text(transcription, lang)
        norm_w = wer(norm_input, norm_output) if norm_input and norm_output else raw_w
        norm_c = cer(norm_input, norm_output) if norm_input and norm_output else raw_c

        return {
            "id": tid, "language": lang, "input_text": text,
            "transcription": transcription,
            "norm_input": norm_input, "norm_output": norm_output,
            "raw_wer": round(raw_w * 100, 2), "raw_cer": round(raw_c * 100, 2),
            "norm_wer": round(norm_w * 100, 2), "norm_cer": round(norm_c * 100, 2),
            "audio_duration_s": round(audio_duration, 2),
            "generation_time_s": round(gen_time / len(cases), 2),
            "rtf": round((gen_time / len(cases)) / audio_duration, 3),
        }

    # Whisper runs on GPU so we do sequential transcription but batch TTS saved time
    t0_whisper = time.time()
    for idx, (wav_path, case) in enumerate(zip(wav_paths, cases)):
        try:
            r = transcribe_one((idx, wav_path, case["text"], case["id"]))
            total_audio_dur += r["audio_duration_s"]
            status = "PERFECT" if r["norm_wer"] == 0 and r["norm_cer"] == 0 else f"WER:{r['norm_wer']:.1f}%"
            print(f"    {r['id']:20s}  {status:10s}  dur:{r['audio_duration_s']:.1f}s")
            results.append(r)
        except Exception as e:
            print(f"    {case['id']:20s}  ERROR: {e}")
            results.append({"id": case["id"], "language": lang, "input_text": case["text"], "error": str(e)})

    whisper_time = time.time() - t0_whisper
    print(f"    Whisper done in {whisper_time:.2f}s")

total_time = time.time() - total_start

# ─────────────────────────────────────────────
# Summary
# ─────────────────────────────────────────────
successful = [r for r in results if "error" not in r]
n = len(successful)

avg_raw_wer = np.mean([r["raw_wer"] for r in successful]) if n else -1
avg_raw_cer = np.mean([r["raw_cer"] for r in successful]) if n else -1
avg_norm_wer = np.mean([r["norm_wer"] for r in successful]) if n else -1
avg_norm_cer = np.mean([r["norm_cer"] for r in successful]) if n else -1

lang_results = defaultdict(lambda: {"raw_wer": [], "raw_cer": [], "norm_wer": [], "norm_cer": [], "rtf": [], "audio_dur": []})
for r in successful:
    for k in ["raw_wer", "raw_cer", "norm_wer", "norm_cer", "rtf"]:
        lang_results[r["language"]][k].append(r[k])
    lang_results[r["language"]]["audio_dur"].append(r["audio_duration_s"])

summary = {
    "model": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    "gpu": "NVIDIA A100-SXM4-80GB",
    "dtype": "bfloat16",
    "mode": "CONCURRENT (batch TTS + sequential Whisper)",
    "whisper_model": "large-v2",
    "total_test_cases": len(test_cases),
    "successful": n,
    "total_time_s": round(total_time, 2),
    "total_tts_gen_time_s": round(total_gen_time, 2),
    "total_audio_duration_s": round(total_audio_dur, 2),
    "overall_rtf": round(total_gen_time / total_audio_dur, 3) if total_audio_dur > 0 else -1,
    "avg_raw_wer_pct": round(avg_raw_wer, 2),
    "avg_raw_cer_pct": round(avg_raw_cer, 2),
    "avg_norm_wer_pct": round(avg_norm_wer, 2),
    "avg_norm_cer_pct": round(avg_norm_cer, 2),
    "per_language": {},
    "results": results,
}

for lang, v in lang_results.items():
    summary["per_language"][lang] = {
        "avg_norm_wer": round(np.mean(v["norm_wer"]), 2),
        "avg_norm_cer": round(np.mean(v["norm_cer"]), 2),
        "avg_raw_wer": round(np.mean(v["raw_wer"]), 2),
        "avg_raw_cer": round(np.mean(v["raw_cer"]), 2),
        "avg_rtf": round(np.mean(v["rtf"]), 3),
        "total_audio_s": round(sum(v["audio_dur"]), 2),
        "n": len(v["rtf"]),
    }

results_path = os.path.join(OUTPUT_DIR, "evaluation_results.json")
with open(results_path, "w") as f:
    json.dump(summary, f, indent=2, ensure_ascii=False)

perfect = sum(1 for r in successful if r["norm_wer"] == 0 and r["norm_cer"] == 0)

print("\n" + "=" * 65)
print("  FINAL RESULTS — Qwen3-TTS-12Hz-0.6B-Base (CONCURRENT)")
print("  Max Accuracy on NVIDIA A100-SXM4-80GB")
print("=" * 65)
print(f"  Precision:     bfloat16 (no quantization)")
print(f"  Mode:          Batch TTS generation per language")
print(f"  Transcriber:   Whisper large-v2 (beam=5, best_of=5)")
print(f"  Test cases:    {n}/{len(test_cases)} successful")
print(f"  Perfect (0%):  {perfect}/{n} ({perfect/n*100:.1f}%)")
print()
print(f"  Timing:")
print(f"    Total wall time:     {total_time:.1f}s")
print(f"    TTS generation:      {total_gen_time:.1f}s")
print(f"    Audio produced:      {total_audio_dur:.1f}s")
print(f"    Overall TTS RTF:     {total_gen_time/total_audio_dur:.3f}" if total_audio_dur > 0 else "")
print()
print(f"  Accuracy (Normalized):")
print(f"    WER:  {avg_norm_wer:.2f}%")
print(f"    CER:  {avg_norm_cer:.2f}%")
print()
print(f"  Accuracy (Raw):")
print(f"    WER:  {avg_raw_wer:.2f}%")
print(f"    CER:  {avg_raw_cer:.2f}%")
print()
print(f"  {'Language':12s}  {'NormWER':>8s}  {'NormCER':>8s}  {'RawWER':>8s}  {'RawCER':>8s}  {'RTF':>6s}  {'n':>3s}")
print(f"  {'-'*12}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*6}  {'-'*3}")
for lang, v in sorted(lang_results.items()):
    print(f"  {lang:12s}  {np.mean(v['norm_wer']):7.2f}%  {np.mean(v['norm_cer']):7.2f}%  {np.mean(v['raw_wer']):7.2f}%  {np.mean(v['raw_cer']):7.2f}%  {np.mean(v['rtf']):5.3f}  {len(v['rtf']):3d}")
print()
print(f"  Results + audio: {OUTPUT_DIR}/")
print("=" * 65)


The file /home/ubuntu/quench_0.6b/eval_tts.py has been updated successfully.

"""
Qwen3-TTS-12Hz-0.6B-Base — Maximum Accuracy Evaluation (CONCURRENT)
GPU: A100-80GB | dtype: bfloat16
Uses batch generation for TTS + concurrent Whisper transcription
"""

import torch
import soundfile as sf
import numpy as np
import os
import re
import time
import json
import whisper
from concurrent.futures import ThreadPoolExecutor
from jiwer import wer, cer
from qwen_tts import Qwen3TTSModel

OUTPUT_DIR = "/home/ubuntu/quench_0.6b/results"
os.makedirs(OUTPUT_DIR, exist_ok=True)

# ─────────────────────────────────────────────
# Text normalization
# ─────────────────────────────────────────────
TRAD_TO_SIMP = str.maketrans(
    "變著們國於華書學過來見與開關認為問題點個這時後從對說無現還進發經資機車門長風東運動當電話開實義氣質員準備區體頭數據覺識邊導彈戰場製議護辦歡環歷歲壓積隊針鑰匙飛網際際計處廣園壞擴聯節職誰認識議導員獨歷歲壓積針飛際計處廣園壞擴聯節職誰導員獨發經過來見與開關認為問題點個這時後從對說無現還進經資機車門長風東運動當電話開實義氣質員準備區體頭數據覺識邊導彈戰場製議護辦歡環歷壓積隊",
    "变着们国于华书学过来见与开关认为问题点个这时后从对说无现还进发经资机车门长风东运动当电话开实义气质员准备区体头数据觉识边导弹战场制议护办欢环历岁压积队针钥匙飞网际际计处广园坏扩联节职谁认识议导员独历岁压积针飞际计处广园坏扩联节职谁导员独发经过来见与开关认为问题点个这时后从对说无现还进经资机车门长风东运动当电话开实义气质员准备区体头数据觉识边导弹战场制议护办欢环历压积队"
)

def normalize_text(text, language="English"):
    t = text.strip()
    t = re.sub(r"['\"""''「」『』（）\(\)]", "", t)
    t = re.sub(r"[,，.。!！?？;；:：、]", " ", t)
    t = re.sub(r"\s+", " ", t).strip().lower()
    if language in ["Chinese", "Japanese"]:
        t = t.translate(TRAD_TO_SIMP)
        t = re.sub(r"\s+", "", t)
    return t

WHISPER_LANG_MAP = {
    "English": "en", "Chinese": "zh", "Japanese": "ja", "Korean": "ko",
    "German": "de", "French": "fr", "Spanish": "es",
}

# ─────────────────────────────────────────────
# Test Sentences
# ─────────────────────────────────────────────
test_cases = [
    {"text": "The quick brown fox jumps over the lazy dog near the riverbank.", "language": "English", "id": "en_pangram"},
    {"text": "Artificial intelligence is transforming how we approach complex problems in science and engineering.", "language": "English", "id": "en_tech"},
    {"text": "The Massachusetts Institute of Technology announced groundbreaking research in superconductivity.", "language": "English", "id": "en_proper_nouns"},
    {"text": "She couldn't believe it and then whispered this changes everything we know about quantum mechanics.", "language": "English", "id": "en_complex"},
    {"text": "Beautiful weather today with clear skies and a gentle breeze coming from the northwest.", "language": "English", "id": "en_weather"},
    {"text": "The orchestra performed a magnificent symphony that left the audience speechless and deeply moved.", "language": "English", "id": "en_arts"},
    {"text": "Renewable energy sources such as solar and wind power are becoming increasingly important worldwide.", "language": "English", "id": "en_energy"},
    {"text": "The philosopher argued that consciousness emerges from the complex interactions of neural networks.", "language": "English", "id": "en_philosophy"},
    {"text": "International cooperation is essential for addressing global challenges like climate change and poverty.", "language": "English", "id": "en_global"},
    {"text": "The restaurant serves traditional Mediterranean cuisine with fresh ingredients sourced from local farmers.", "language": "English", "id": "en_food"},
    {"text": "人工智能正在深刻改变着我们的日常生活和工作方式。", "language": "Chinese", "id": "zh_ai"},
    {"text": "今天天气非常好，蓝天白云，微风轻拂。", "language": "Chinese", "id": "zh_weather"},
    {"text": "科学技术是第一生产力，创新驱动发展。", "language": "Chinese", "id": "zh_science"},
    {"text": "这座城市有着悠久的历史和丰富的文化遗产。", "language": "Chinese", "id": "zh_culture"},
    {"text": "東京は日本の首都であり、世界で最も人口の多い都市の一つです。", "language": "Japanese", "id": "ja_tokyo"},
    {"text": "桜の花が満開で、とても美しい景色が広がっています。", "language": "Japanese", "id": "ja_cherry"},
    {"text": "인공지능 기술은 우리의 일상생활을 크게 변화시키고 있습니다.", "language": "Korean", "id": "ko_ai"},
    {"text": "오늘 날씨가 정말 좋아서 산책하기에 딱 좋은 날입니다.", "language": "Korean", "id": "ko_weather"},
    {"text": "Die Wissenschaft hat in den letzten Jahren enorme Fortschritte gemacht.", "language": "German", "id": "de_science"},
    {"text": "La technologie transforme notre façon de vivre et de travailler au quotidien.", "language": "French", "id": "fr_tech"},
    {"text": "La inteligencia artificial está revolucionando todos los aspectos de nuestra vida cotidiana.", "language": "Spanish", "id": "es_ai"},
]

REF_AUDIO = "/home/ubuntu/quench_0.6b/ref_audio.wav"
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."

# ─────────────────────────────────────────────
# Load Models
# ─────────────────────────────────────────────
print("=" * 60)
print("  Qwen3-TTS-12Hz-0.6B-Base — CONCURRENT Max Accuracy Eval")
print("=" * 60)

print("\n[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...")
t0 = time.time()
tts_model = Qwen3TTSModel.from_pretrained(
    "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    device_map="cuda:0",
    dtype=torch.bfloat16,
)
# Pre-cache the voice clone prompt once (reuse for all samples)
voice_prompt = tts_model.create_voice_clone_prompt(
    ref_audio=REF_AUDIO,
    ref_text=REF_TEXT,
)
print(f"  TTS model + voice prompt cached in {time.time()-t0:.1f}s")

print("\n[2/2] Loading Whisper large-v2 for transcription...")
t0 = time.time()
whisper_model = whisper.load_model("large-v2", device="cuda:0")
print(f"  Whisper loaded in {time.time()-t0:.1f}s")

# ─────────────────────────────────────────────
# CONCURRENT Evaluation
# Group by language for batch TTS generation
# ─────────────────────────────────────────────
print(f"\nRunning CONCURRENT evaluation on {len(test_cases)} test cases...")
print("-" * 60)

# Group test cases by language for batched TTS
from collections import defaultdict
lang_groups = defaultdict(list)
for tc in test_cases:
    lang_groups[tc["language"]].append(tc)

results = []
total_gen_time = 0
total_audio_dur = 0
total_start = time.time()

for lang, cases in lang_groups.items():
    batch_texts = [c["text"] for c in cases]
    batch_langs = [lang] * len(cases)
    batch_ids = [c["id"] for c in cases]

    print(f"\n  BATCH [{lang}] — {len(cases)} samples")

    # Batch TTS generation
    t0 = time.time()
    try:
        wavs, sr = tts_model.generate_voice_clone(
            text=batch_texts,
            language=batch_langs,
            voice_clone_prompt=voice_prompt,
            non_streaming_mode=True,
        )
        gen_time = time.time() - t0
        total_gen_time += gen_time
        print(f"    Batch TTS done in {gen_time:.2f}s ({len(cases)} samples)")
    except Exception as e:
        print(f"    BATCH TTS ERROR: {e}")
        import traceback; traceback.print_exc()
        for c in cases:
            results.append({"id": c["id"], "language": lang, "input_text": c["text"], "error": str(e)})
        continue

    # Save all wav files
    wav_paths = []
    for i, (w, tid) in enumerate(zip(wavs, batch_ids)):
        wav_path = os.path.join(OUTPUT_DIR, f"{tid}.wav")
        sf.write(wav_path, w, sr)
        wav_paths.append(wav_path)

    # Concurrent Whisper transcription (CPU-bound prep is parallelizable)
    def transcribe_one(args):
        idx, wav_path, text, tid = args
        audio = wavs[idx]
        audio_duration = len(audio) / sr

        whisper_result = whisper_model.transcribe(
            wav_path,
            language=WHISPER_LANG_MAP.get(lang, "en"),
            beam_size=5,
            best_of=5,
            temperature=0.0,
        )
        transcription = whisper_result["text"].strip()

        raw_w = wer(text, transcription)
        raw_c = cer(text, transcription)
        norm_input = normalize_text(text, lang)
        norm_output = normalize_text(transcription, lang)
        norm_w = wer(norm_input, norm_output) if norm_input and norm_output else raw_w
        norm_c = cer(norm_input, norm_output) if norm_input and norm_output else raw_c

        return {
            "id": tid, "language": lang, "input_text": text,
            "transcription": transcription,
            "norm_input": norm_input, "norm_output": norm_output,
            "raw_wer": round(raw_w * 100, 2), "raw_cer": round(raw_c * 100, 2),
            "norm_wer": round(norm_w * 100, 2), "norm_cer": round(norm_c * 100, 2),
            "audio_duration_s": round(audio_duration, 2),
            "generation_time_s": round(gen_time / len(cases), 2),
            "rtf": round((gen_time / len(cases)) / audio_duration, 3),
        }

    # Whisper runs on GPU so we do sequential transcription but batch TTS saved time
    t0_whisper = time.time()
    for idx, (wav_path, case) in enumerate(zip(wav_paths, cases)):
        try:
            r = transcribe_one((idx, wav_path, case["text"], case["id"]))
            total_audio_dur += r["audio_duration_s"]
            status = "PERFECT" if r["norm_wer"] == 0 and r["norm_cer"] == 0 else f"WER:{r['norm_wer']:.1f}%"
            print(f"    {r['id']:20s}  {status:10s}  dur:{r['audio_duration_s']:.1f}s")
            results.append(r)
        except Exception as e:
            print(f"    {case['id']:20s}  ERROR: {e}")
            results.append({"id": case["id"], "language": lang, "input_text": case["text"], "error": str(e)})

    whisper_time = time.time() - t0_whisper
    print(f"    Whisper done in {whisper_time:.2f}s")

total_time = time.time() - total_start

# ─────────────────────────────────────────────
# Summary
# ─────────────────────────────────────────────
successful = [r for r in results if "error" not in r]
n = len(successful)

avg_raw_wer = np.mean([r["raw_wer"] for r in successful]) if n else -1
avg_raw_cer = np.mean([r["raw_cer"] for r in successful]) if n else -1
avg_norm_wer = np.mean([r["norm_wer"] for r in successful]) if n else -1
avg_norm_cer = np.mean([r["norm_cer"] for r in successful]) if n else -1

lang_results = defaultdict(lambda: {"raw_wer": [], "raw_cer": [], "norm_wer": [], "norm_cer": [], "rtf": [], "audio_dur": []})
for r in successful:
    for k in ["raw_wer", "raw_cer", "norm_wer", "norm_cer", "rtf"]:
        lang_results[r["language"]][k].append(r[k])
    lang_results[r["language"]]["audio_dur"].append(r["audio_duration_s"])

summary = {
    "model": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    "gpu": "NVIDIA A100-SXM4-80GB",
    "dtype": "bfloat16",
    "mode": "CONCURRENT (batch TTS + sequential Whisper)",
    "whisper_model": "large-v2",
    "total_test_cases": len(test_cases),
    "successful": n,
    "total_time_s": round(total_time, 2),
    "total_tts_gen_time_s": round(total_gen_time, 2),
    "total_audio_duration_s": round(total_audio_dur, 2),
    "overall_rtf": round(total_gen_time / total_audio_dur, 3) if total_audio_dur > 0 else -1,
    "avg_raw_wer_pct": round(avg_raw_wer, 2),
    "avg_raw_cer_pct": round(avg_raw_cer, 2),
    "avg_norm_wer_pct": round(avg_norm_wer, 2),
    "avg_norm_cer_pct": round(avg_norm_cer, 2),
    "per_language": {},
    "results": results,
}

for lang, v in lang_results.items():
    summary["per_language"][lang] = {
        "avg_norm_wer": round(np.mean(v["norm_wer"]), 2),
        "avg_norm_cer": round(np.mean(v["norm_cer"]), 2),
        "avg_raw_wer": round(np.mean(v["raw_wer"]), 2),
        "avg_raw_cer": round(np.mean(v["raw_cer"]), 2),
        "avg_rtf": round(np.mean(v["rtf"]), 3),
        "total_audio_s": round(sum(v["audio_dur"]), 2),
        "n": len(v["rtf"]),
    }

results_path = os.path.join(OUTPUT_DIR, "evaluation_results.json")
with open(results_path, "w") as f:
    json.dump(summary, f, indent=2, ensure_ascii=False)

perfect = sum(1 for r in successful if r["norm_wer"] == 0 and r["norm_cer"] == 0)

print("\n" + "=" * 65)
print("  FINAL RESULTS — Qwen3-TTS-12Hz-0.6B-Base (CONCURRENT)")
print("  Max Accuracy on NVIDIA A100-SXM4-80GB")
print("=" * 65)
print(f"  Precision:     bfloat16 (no quantization)")
print(f"  Mode:          Batch TTS generation per language")
print(f"  Transcriber:   Whisper large-v2 (beam=5, best_of=5)")
print(f"  Test cases:    {n}/{len(test_cases)} successful")
print(f"  Perfect (0%):  {perfect}/{n} ({perfect/n*100:.1f}%)")
print()
print(f"  Timing:")
print(f"    Total wall time:     {total_time:.1f}s")
print(f"    TTS generation:      {total_gen_time:.1f}s")
print(f"    Audio produced:      {total_audio_dur:.1f}s")
print(f"    Overall TTS RTF:     {total_gen_time/total_audio_dur:.3f}" if total_audio_dur > 0 else "")
print()
print(f"  Accuracy (Normalized):")
print(f"    WER:  {avg_norm_wer:.2f}%")
print(f"    CER:  {avg_norm_cer:.2f}%")
print()
print(f"  Accuracy (Raw):")
print(f"    WER:  {avg_raw_wer:.2f}%")
print(f"    CER:  {avg_raw_cer:.2f}%")
print()
print(f"  {'Language':12s}  {'NormWER':>8s}  {'NormCER':>8s}  {'RawWER':>8s}  {'RawCER':>8s}  {'RTF':>6s}  {'n':>3s}")
print(f"  {'-'*12}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*6}  {'-'*3}")
for lang, v in sorted(lang_results.items()):
    print(f"  {lang:12s}  {np.mean(v['norm_wer']):7.2f}%  {np.mean(v['norm_cer']):7.2f}%  {np.mean(v['raw_wer']):7.2f}%  {np.mean(v['raw_cer']):7.2f}%  {np.mean(v['rtf']):5.3f}  {len(v['rtf']):3d}")
print()
print(f"  Results + audio: {OUTPUT_DIR}/")
print("=" * 65)

-Qwen3-TTS-12Hz-0.6B-Base — Maximum Accuracy Evaluation

+Qwen3-TTS-12Hz-0.6B-Base — Maximum Accuracy Evaluation (CONCURRENT)

-Metrics: Word Error Rate (WER), Character Error Rate (CER)

-With proper text normalization to account for Whisper formatting

+Uses batch generation for TTS + concurrent Whisper transcription

+from concurrent.futures import ThreadPoolExecutor

-# Text normalization for fair TTS evaluation

-# Traditional -> Simplified Chinese mapping (common chars)

-    """Normalize text for fair WER/CER comparison between input and Whisper output."""

-    # Remove punctuation that Whisper may drop or change

t = re.sub(r"\s+", " ", t).strip().lower()

+    "English": "en", "Chinese": "zh", "Japanese": "ja", "Korean": "ko",

+    "German": "de", "French": "fr", "Spanish": "es",

-print("  Qwen3-TTS-12Hz-0.6B-Base — Max Accuracy Eval")

-print("\n[1/3] Loading Qwen3-TTS model (bfloat16, CUDA)...")

-print(f"  TTS model loaded in {time.time()-t0:.1f}s")

-print("\n[2/3] Loading Whisper large-v2 for transcription...")

-whisper_model = whisper.load_model("large-v2", device="cuda:0")

-print(f"  Whisper loaded in {time.time()-t0:.1f}s")

-    # English - clean text (no numbers to avoid Whisper digit formatting)

-    "English": "en", "Chinese": "zh", "Japanese": "ja", "Korean": "ko",

-    "German": "de", "French": "fr", "Spanish": "es",

+print("  Qwen3-TTS-12Hz-0.6B-Base — CONCURRENT Max Accuracy Eval")

+print("\n[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...")

+# Pre-cache the voice clone prompt once (reuse for all samples)

+print(f"  TTS model + voice prompt cached in {time.time()-t0:.1f}s")

+print("\n[2/2] Loading Whisper large-v2 for transcription...")

+whisper_model = whisper.load_model("large-v2", device="cuda:0")

+print(f"  Whisper loaded in {time.time()-t0:.1f}s")

+# Group by language for batch TTS generation

-print(f"\n[3/3] Running evaluation on {len(test_cases)} test cases...")

+print(f"\nRunning CONCURRENT evaluation on {len(test_cases)} test cases...")

+# Group test cases by language for batched TTS

+    lang_groups[tc["language"]].append(tc)

+for lang, cases in lang_groups.items():

+    batch_texts = [c["text"] for c in cases]

+    batch_ids = [c["id"] for c in cases]

-    print(f"\n  [{i+1}/{len(test_cases)}] {tid} ({lang})")

-    print(f"    Input : {text[:80]}{'...' if len(text)>80 else ''}")

+    print(f"\n  BATCH [{lang}] — {len(cases)} samples")

wavs, sr = tts_model.generate_voice_clone(

+            voice_clone_prompt=voice_prompt,

+        print(f"    Batch TTS done in {gen_time:.2f}s ({len(cases)} samples)")

+        print(f"    BATCH TTS ERROR: {e}")

+        import traceback; traceback.print_exc()

+            results.append({"id": c["id"], "language": lang, "input_text": c["text"], "error": str(e)})

+    for i, (w, tid) in enumerate(zip(wavs, batch_ids)):

wav_path = os.path.join(OUTPUT_DIR, f"{tid}.wav")

+    # Concurrent Whisper transcription (CPU-bound prep is parallelizable)

+        idx, wav_path, text, tid = args

-        rtf = gen_time / audio_duration

-        # Normalized metrics (fair comparison)

norm_output = normalize_text(transcription, lang)

-        print(f"    Output: {transcription[:80]}{'...' if len(transcription)>80 else ''}")

-        print(f"    Audio: {audio_duration:.2f}s | Gen: {gen_time:.2f}s | RTF: {rtf:.3f}")

"id": tid, "language": lang, "input_text": text,

"norm_input": norm_input, "norm_output": norm_output,

-            "generation_time_s": round(gen_time, 2),

-        import traceback; traceback.print_exc()

-        results.append({"id": tid, "language": lang, "input_text": text, "error": str(e)})

+            "generation_time_s": round(gen_time / len(cases), 2),

+            "rtf": round((gen_time / len(cases)) / audio_duration, 3),

+    # Whisper runs on GPU so we do sequential transcription but batch TTS saved time

+    for idx, (wav_path, case) in enumerate(zip(wav_paths, cases)):

+            r = transcribe_one((idx, wav_path, case["text"], case["id"]))

+            total_audio_dur += r["audio_duration_s"]

+            status = "PERFECT" if r["norm_wer"] == 0 and r["norm_cer"] == 0 else f"WER:{r['norm_wer']:.1f}%"

+            print(f"    {r['id']:20s}  {status:10s}  dur:{r['audio_duration_s']:.1f}s")

+            print(f"    {case['id']:20s}  ERROR: {e}")

+            results.append({"id": case["id"], "language": lang, "input_text": case["text"], "error": str(e)})

+    whisper_time = time.time() - t0_whisper

+    print(f"    Whisper done in {whisper_time:.2f}s")

successful = [r for r in results if "error" not in r]

-avg_rtf = np.mean([r["rtf"] for r in successful]) if n else -1

+lang_results = defaultdict(lambda: {"raw_wer": [], "raw_cer": [], "norm_wer": [], "norm_cer": [], "rtf": [], "audio_dur": []})

-        lang_results[lang] = {"raw_wer": [], "raw_cer": [], "norm_wer": [], "norm_cer": [], "rtf": []}

for k in ["raw_wer", "raw_cer", "norm_wer", "norm_cer", "rtf"]:

-        lang_results[lang][k].append(r[k])

+        lang_results[r["language"]][k].append(r[k])

+    lang_results[r["language"]]["audio_dur"].append(r["audio_duration_s"])

+    "mode": "CONCURRENT (batch TTS + sequential Whisper)",

+    "total_time_s": round(total_time, 2),

+    "total_tts_gen_time_s": round(total_gen_time, 2),

+    "total_audio_duration_s": round(total_audio_dur, 2),

+    "overall_rtf": round(total_gen_time / total_audio_dur, 3) if total_audio_dur > 0 else -1,

-for lang, vals in lang_results.items():

-        "avg_raw_wer": round(np.mean(vals["raw_wer"]), 2),

-        "avg_raw_cer": round(np.mean(vals["raw_cer"]), 2),

-        "avg_norm_wer": round(np.mean(vals["norm_wer"]), 2),

-        "avg_norm_cer": round(np.mean(vals["norm_cer"]), 2),

-        "avg_rtf": round(np.mean(vals["rtf"]), 3),

+        "avg_norm_wer": round(np.mean(v["norm_wer"]), 2),

+        "avg_norm_cer": round(np.mean(v["norm_cer"]), 2),

+        "avg_raw_wer": round(np.mean(v["raw_wer"]), 2),

+        "avg_raw_cer": round(np.mean(v["raw_cer"]), 2),

+        "avg_rtf": round(np.mean(v["rtf"]), 3),

+        "total_audio_s": round(sum(v["audio_dur"]), 2),

json.dump(summary, f, indent=2, ensure_ascii=False)

-print("  FINAL RESULTS — Qwen3-TTS-12Hz-0.6B-Base")

+perfect = sum(1 for r in successful if r["norm_wer"] == 0 and r["norm_cer"] == 0)

+print("  FINAL RESULTS — Qwen3-TTS-12Hz-0.6B-Base (CONCURRENT)")

print("  Max Accuracy on NVIDIA A100-SXM4-80GB")

print(f"  Precision:     bfloat16 (no quantization)")

+print(f"  Mode:          Batch TTS generation per language")

print(f"  Transcriber:   Whisper large-v2 (beam=5, best_of=5)")

print(f"  Test cases:    {n}/{len(test_cases)} successful")

+print(f"  Perfect (0%):  {perfect}/{n} ({perfect/n*100:.1f}%)")

-print(f"  Overall (Normalized - fair comparison):")

+print(f"    Total wall time:     {total_time:.1f}s")

+print(f"    TTS generation:      {total_gen_time:.1f}s")

+print(f"    Audio produced:      {total_audio_dur:.1f}s")

+print(f"    Overall TTS RTF:     {total_gen_time/total_audio_dur:.3f}" if total_audio_dur > 0 else "")

-print(f"  Overall (Raw - includes formatting diffs):")

for lang, v in sorted(lang_results.items()):

-print(f"  Results + audio saved to: {OUTPUT_DIR}/")

+print(f"  Results + audio: {OUTPUT_DIR}/")

"""
Qwen3-TTS-12Hz-0.6B-Base — Maximum Accuracy Evaluation
GPU: A100-80GB | dtype: bfloat16
Metrics: Word Error Rate (WER), Character Error Rate (CER)
With proper text normalization to account for Whisper formatting
"""

import torch
import soundfile as sf
import numpy as np
import os
import re
import time
import json
import whisper
from jiwer import wer, cer
from qwen_tts import Qwen3TTSModel

OUTPUT_DIR = "/home/ubuntu/quench_0.6b/results"
os.makedirs(OUTPUT_DIR, exist_ok=True)

# ─────────────────────────────────────────────
# Text normalization for fair TTS evaluation
# ─────────────────────────────────────────────
NUM_WORDS = {
    "zero": "0", "one": "1", "two": "2", "three": "3", "four": "4",
    "five": "5", "six": "6", "seven": "7", "eight": "8", "nine": "9",
    "ten": "10", "eleven": "11", "twelve": "12", "thirteen": "13",
    "fourteen": "14", "fifteen": "15", "sixteen": "16", "seventeen": "17",
    "eighteen": "18", "nineteen": "19", "twenty": "20", "thirty": "30",
    "forty": "40", "fifty": "50", "sixty": "60", "seventy": "70",
    "eighty": "80", "ninety": "90", "hundred": "100", "thousand": "1000",
    "million": "1000000", "billion": "1000000000",
}

# Traditional -> Simplified Chinese mapping (common chars)
TRAD_TO_SIMP = str.maketrans(
    "變著們國於華書學過來見與開關認為問題點個這時後從對說無現還進發經資機車門長風東運動當電話開實義氣質員準備區體頭數據覺識邊導彈戰場製議護辦歡環歷歲壓積隊針鑰匙飛網際際計處廣園壞擴聯節職誰認識議導員獨歷歲壓積針飛際計處廣園壞擴聯節職誰導員獨發經過來見與開關認為問題點個這時後從對說無現還進經資機車門長風東運動當電話開實義氣質員準備區體頭數據覺識邊導彈戰場製議護辦歡環歷壓積隊",
    "变着们国于华书学过来见与开关认为问题点个这时后从对说无现还进发经资机车门长风东运动当电话开实义气质员准备区体头数据觉识边导弹战场制议护办欢环历岁压积队针钥匙飞网际际计处广园坏扩联节职谁认识议导员独历岁压积针飞际计处广园坏扩联节职谁导员独发经过来见与开关认为问题点个这时后从对说无现还进经资机车门长风东运动当电话开实义气质员准备区体头数据觉识边导弹战场制议护办欢环历压积队"
)

def normalize_text(text, language="English"):
    """Normalize text for fair WER/CER comparison between input and Whisper output."""
    t = text.strip()
    # Remove punctuation that Whisper may drop or change
    t = re.sub(r"['\"""''「」『』（）\(\)]", "", t)
    t = re.sub(r"[,，.。!！?？;；:：、]", " ", t)
    t = re.sub(r"\s+", " ", t).strip().lower()

    if language in ["Chinese", "Japanese"]:
        # Traditional -> Simplified
        t = t.translate(TRAD_TO_SIMP)
        # Remove all spaces for CJK
        t = re.sub(r"\s+", "", t)

    return t

# ─────────────────────────────────────────────
# 1. Load Models
# ─────────────────────────────────────────────
print("=" * 60)
print("  Qwen3-TTS-12Hz-0.6B-Base — Max Accuracy Eval")
print("=" * 60)

print("\n[1/3] Loading Qwen3-TTS model (bfloat16, CUDA)...")
t0 = time.time()
tts_model = Qwen3TTSModel.from_pretrained(
    "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    device_map="cuda:0",
    dtype=torch.bfloat16,
)
print(f"  TTS model loaded in {time.time()-t0:.1f}s")

print("\n[2/3] Loading Whisper large-v2 for transcription...")
t0 = time.time()
whisper_model = whisper.load_model("large-v2", device="cuda:0")
print(f"  Whisper loaded in {time.time()-t0:.1f}s")

# ─────────────────────────────────────────────
# 2. Test Sentences
# ─────────────────────────────────────────────
test_cases = [
    # English - clean text (no numbers to avoid Whisper digit formatting)
    {"text": "The quick brown fox jumps over the lazy dog near the riverbank.", "language": "English", "id": "en_pangram"},
    {"text": "Artificial intelligence is transforming how we approach complex problems in science and engineering.", "language": "English", "id": "en_tech"},
    {"text": "The Massachusetts Institute of Technology announced groundbreaking research in superconductivity.", "language": "English", "id": "en_proper_nouns"},
    {"text": "She couldn't believe it and then whispered this changes everything we know about quantum mechanics.", "language": "English", "id": "en_complex"},
    {"text": "Beautiful weather today with clear skies and a gentle breeze coming from the northwest.", "language": "English", "id": "en_weather"},
    {"text": "The orchestra performed a magnificent symphony that left the audience speechless and deeply moved.", "language": "English", "id": "en_arts"},
    {"text": "Renewable energy sources such as solar and wind power are becoming increasingly important worldwide.", "language": "English", "id": "en_energy"},
    {"text": "The philosopher argued that consciousness emerges from the complex interactions of neural networks.", "language": "English", "id": "en_philosophy"},
    {"text": "International cooperation is essential for addressing global challenges like climate change and poverty.", "language": "English", "id": "en_global"},
    {"text": "The restaurant serves traditional Mediterranean cuisine with fresh ingredients sourced from local farmers.", "language": "English", "id": "en_food"},
    # Chinese
    {"text": "人工智能正在深刻改变着我们的日常生活和工作方式。", "language": "Chinese", "id": "zh_ai"},
    {"text": "今天天气非常好，蓝天白云，微风轻拂。", "language": "Chinese", "id": "zh_weather"},
    {"text": "科学技术是第一生产力，创新驱动发展。", "language": "Chinese", "id": "zh_science"},
    {"text": "这座城市有着悠久的历史和丰富的文化遗产。", "language": "Chinese", "id": "zh_culture"},
    # Japanese
    {"text": "東京は日本の首都であり、世界で最も人口の多い都市の一つです。", "language": "Japanese", "id": "ja_tokyo"},
    {"text": "桜の花が満開で、とても美しい景色が広がっています。", "language": "Japanese", "id": "ja_cherry"},
    # Korean
    {"text": "인공지능 기술은 우리의 일상생활을 크게 변화시키고 있습니다.", "language": "Korean", "id": "ko_ai"},
    {"text": "오늘 날씨가 정말 좋아서 산책하기에 딱 좋은 날입니다.", "language": "Korean", "id": "ko_weather"},
    # German
    {"text": "Die Wissenschaft hat in den letzten Jahren enorme Fortschritte gemacht.", "language": "German", "id": "de_science"},
    # French
    {"text": "La technologie transforme notre façon de vivre et de travailler au quotidien.", "language": "French", "id": "fr_tech"},
    # Spanish
    {"text": "La inteligencia artificial está revolucionando todos los aspectos de nuestra vida cotidiana.", "language": "Spanish", "id": "es_ai"},
]

REF_AUDIO = "/home/ubuntu/quench_0.6b/ref_audio.wav"
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."

WHISPER_LANG_MAP = {
    "English": "en", "Chinese": "zh", "Japanese": "ja", "Korean": "ko",
    "German": "de", "French": "fr", "Spanish": "es",
}

# ─────────────────────────────────────────────
# 3. Run Evaluation
# ─────────────────────────────────────────────
print(f"\n[3/3] Running evaluation on {len(test_cases)} test cases...")
print("-" * 60)

results = []

for i, tc in enumerate(test_cases):
    text = tc["text"]
    lang = tc["language"]
    tid = tc["id"]

    print(f"\n  [{i+1}/{len(test_cases)}] {tid} ({lang})")
    print(f"    Input : {text[:80]}{'...' if len(text)>80 else ''}")

    t0 = time.time()
    try:
        wavs, sr = tts_model.generate_voice_clone(
            text=text,
            language=lang,
            ref_audio=REF_AUDIO,
            ref_text=REF_TEXT,
            non_streaming_mode=True,
        )
        gen_time = time.time() - t0
        audio = wavs[0]

        wav_path = os.path.join(OUTPUT_DIR, f"{tid}.wav")
        sf.write(wav_path, audio, sr)

        audio_duration = len(audio) / sr
        rtf = gen_time / audio_duration

        whisper_result = whisper_model.transcribe(
            wav_path,
            language=WHISPER_LANG_MAP.get(lang, "en"),
            beam_size=5,
            best_of=5,
            temperature=0.0,
        )
        transcription = whisper_result["text"].strip()

        # Raw metrics
        raw_w = wer(text, transcription)
        raw_c = cer(text, transcription)

        # Normalized metrics (fair comparison)
        norm_input = normalize_text(text, lang)
        norm_output = normalize_text(transcription, lang)
        norm_w = wer(norm_input, norm_output) if norm_input and norm_output else raw_w
        norm_c = cer(norm_input, norm_output) if norm_input and norm_output else raw_c

        print(f"    Output: {transcription[:80]}{'...' if len(transcription)>80 else ''}")
        print(f"    Raw    WER: {raw_w*100:6.2f}%  CER: {raw_c*100:6.2f}%")
        print(f"    Norm   WER: {norm_w*100:6.2f}%  CER: {norm_c*100:6.2f}%")
        print(f"    Audio: {audio_duration:.2f}s | Gen: {gen_time:.2f}s | RTF: {rtf:.3f}")

        results.append({
            "id": tid, "language": lang, "input_text": text,
            "transcription": transcription,
            "norm_input": norm_input, "norm_output": norm_output,
            "raw_wer": round(raw_w * 100, 2), "raw_cer": round(raw_c * 100, 2),
            "norm_wer": round(norm_w * 100, 2), "norm_cer": round(norm_c * 100, 2),
            "audio_duration_s": round(audio_duration, 2),
            "generation_time_s": round(gen_time, 2),
            "rtf": round(rtf, 3),
        })
    except Exception as e:
        print(f"    ERROR: {e}")
        import traceback; traceback.print_exc()
        results.append({"id": tid, "language": lang, "input_text": text, "error": str(e)})

# ─────────────────────────────────────────────
# 4. Summary
# ─────────────────────────────────────────────
successful = [r for r in results if "error" not in r]
n = len(successful)

avg_raw_wer = np.mean([r["raw_wer"] for r in successful]) if n else -1
avg_raw_cer = np.mean([r["raw_cer"] for r in successful]) if n else -1
avg_norm_wer = np.mean([r["norm_wer"] for r in successful]) if n else -1
avg_norm_cer = np.mean([r["norm_cer"] for r in successful]) if n else -1
avg_rtf = np.mean([r["rtf"] for r in successful]) if n else -1

# Per-language breakdown
lang_results = {}
for r in successful:
    lang = r["language"]
    if lang not in lang_results:
        lang_results[lang] = {"raw_wer": [], "raw_cer": [], "norm_wer": [], "norm_cer": [], "rtf": []}
    for k in ["raw_wer", "raw_cer", "norm_wer", "norm_cer", "rtf"]:
        lang_results[lang][k].append(r[k])

summary = {
    "model": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    "gpu": "NVIDIA A100-SXM4-80GB",
    "dtype": "bfloat16",
    "whisper_model": "large-v2",
    "total_test_cases": len(test_cases),
    "successful": n,
    "avg_raw_wer_pct": round(avg_raw_wer, 2),
    "avg_raw_cer_pct": round(avg_raw_cer, 2),
    "avg_norm_wer_pct": round(avg_norm_wer, 2),
    "avg_norm_cer_pct": round(avg_norm_cer, 2),
    "avg_rtf": round(avg_rtf, 3),
    "per_language": {},
    "results": results,
}

for lang, vals in lang_results.items():
    summary["per_language"][lang] = {
        "avg_raw_wer": round(np.mean(vals["raw_wer"]), 2),
        "avg_raw_cer": round(np.mean(vals["raw_cer"]), 2),
        "avg_norm_wer": round(np.mean(vals["norm_wer"]), 2),
        "avg_norm_cer": round(np.mean(vals["norm_cer"]), 2),
        "avg_rtf": round(np.mean(vals["rtf"]), 3),
        "n": len(vals["rtf"]),
    }

results_path = os.path.join(OUTPUT_DIR, "evaluation_results.json")
with open(results_path, "w") as f:
    json.dump(summary, f, indent=2, ensure_ascii=False)

print("\n" + "=" * 60)
print("  FINAL RESULTS — Qwen3-TTS-12Hz-0.6B-Base")
print("  Max Accuracy on NVIDIA A100-SXM4-80GB")
print("=" * 60)
print(f"  Precision:     bfloat16 (no quantization)")
print(f"  Transcriber:   Whisper large-v2 (beam=5, best_of=5)")
print(f"  Test cases:    {n}/{len(test_cases)} successful")
print()
print(f"  Overall (Normalized - fair comparison):")
print(f"    WER:  {avg_norm_wer:.2f}%")
print(f"    CER:  {avg_norm_cer:.2f}%")
print(f"    RTF:  {avg_rtf:.3f}")
print()
print(f"  Overall (Raw - includes formatting diffs):")
print(f"    WER:  {avg_raw_wer:.2f}%")
print(f"    CER:  {avg_raw_cer:.2f}%")
print()
print(f"  {'Language':12s}  {'NormWER':>8s}  {'NormCER':>8s}  {'RawWER':>8s}  {'RawCER':>8s}  {'RTF':>6s}  {'n':>3s}")
print(f"  {'-'*12}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*6}  {'-'*3}")
for lang, v in sorted(lang_results.items()):
    print(f"  {lang:12s}  {np.mean(v['norm_wer']):7.2f}%  {np.mean(v['norm_cer']):7.2f}%  {np.mean(v['raw_wer']):7.2f}%  {np.mean(v['raw_cer']):7.2f}%  {np.mean(v['rtf']):5.3f}  {len(v['rtf']):3d}")
print()
print(f"  Results + audio saved to: {OUTPUT_DIR}/")
print("=" * 60)


Bash

Run concurrent TTS evaluation with batch generation


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

********
Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference.
********

============================================================
  Qwen3-TTS-12Hz-0.6B-Base — CONCURRENT Max Accuracy Eval
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

********
Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference.
********

============================================================
  Qwen3-TTS-12Hz-0.6B-Base — CONCURRENT Max Accuracy Eval
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

********
Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference.
********

============================================================
  Qwen3-TTS-12Hz-0.6B-Base — CONCURRENT Max Accuracy Eval
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

********
Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference.
********

============================================================
  Qwen3-TTS-12Hz-0.6B-Base — CONCURRENT Max Accuracy Eval
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

********
Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference.
********

============================================================
  Qwen3-TTS-12Hz-0.6B-Base — CONCURRENT Max Accuracy Eval
============================================================

[1/2] Loading Qwen3-TTS model (bfloat16, CUDA)...

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 63072.24it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.