The file /home/ubuntu/quench_0.6b/load_test.py has been updated. All occurrences of '    chunks = list(fast_model2.generate_voice_clone_streaming(
        text="Warmup.", language="English",
        ref_audio=REF_AUDIO, ref_text=REF_TEXT,
    ))' were successfully replaced with '    for _ in fast_model2.generate_voice_clone_streaming(
        text="Warmup.", language="English",
        ref_audio=REF_AUDIO, ref_text=REF_TEXT,
    ):
        pass'.

"""
Qwen3-TTS-12Hz-0.6B-Base — Production Load Test
Simulates concurrent voice agent users hitting TTS
Measures: TTFB, E2E latency, throughput, concurrency limits
"""

import torch
import numpy as np
import os
import time
import json
import threading
import queue
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, asdict
from typing import List

OUTPUT_DIR = "/home/ubuntu/quench_0.6b/results"
os.makedirs(OUTPUT_DIR, exist_ok=True)

REF_AUDIO = "/home/ubuntu/quench_0.6b/ref_audio.wav"
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."

# Realistic voice agent sentences (varying lengths)
AGENT_SENTENCES = [
    "Hi there, how can I help you today?",
    "Sure, let me look that up for you right away.",
    "Your order has been confirmed and will arrive within three to five business days.",
    "I understand your concern. Let me transfer you to a specialist who can help.",
    "The total comes to forty seven dollars and sixty three cents. Would you like to proceed?",
    "Thank you for calling. Is there anything else I can assist you with?",
    "I'm sorry to hear about that experience. Let me see what I can do to resolve this.",
    "Your appointment is scheduled for next Tuesday at two thirty in the afternoon.",
    "I've updated your account information. You should receive a confirmation email shortly.",
    "Please hold for just a moment while I check the availability.",
    "Great news, we have that item in stock and it can be shipped today.",
    "For security purposes, could you please verify your date of birth?",
    "The warranty on your product covers manufacturing defects for up to two years.",
    "I've applied a fifteen percent discount to your order as a courtesy.",
    "Our business hours are Monday through Friday, nine AM to six PM Eastern time.",
]

@dataclass
class RequestResult:
    request_id: int
    concurrency_level: int
    sentence: str
    ttfb_ms: float  # time to first audio chunk (streaming)
    e2e_latency_ms: float  # total generation time
    audio_duration_s: float
    rtf: float
    success: bool
    error: str = ""

def run_single_request(model, voice_prompt, request_id, concurrency_level):
    """Simulate one voice agent TTS request."""
    sentence = AGENT_SENTENCES[request_id % len(AGENT_SENTENCES)]

    try:
        # Measure TTFB using streaming mode
        torch.cuda.synchronize()
        t_start = time.perf_counter()
        ttfb = None

        # Use streaming to get TTFB
        chunks = []
        sr = None
        for chunk_data in model.generate_voice_clone_streaming(
            text=sentence,
            language="English",
            ref_audio=REF_AUDIO,
            ref_text=REF_TEXT,
        ):
            chunk_wav = chunk_data[0]
            chunk_sr = chunk_data[1]
            if ttfb is None:
                torch.cuda.synchronize()
                ttfb = (time.perf_counter() - t_start) * 1000  # ms
            chunks.append(chunk_wav)
            sr = chunk_sr

        torch.cuda.synchronize()
        e2e = (time.perf_counter() - t_start) * 1000  # ms

        if chunks and sr:
            full_audio = np.concatenate(chunks)
            audio_dur = len(full_audio) / sr
            rtf = (e2e / 1000) / audio_dur
        else:
            audio_dur = 0
            rtf = float('inf')

        return RequestResult(
            request_id=request_id,
            concurrency_level=concurrency_level,
            sentence=sentence,
            ttfb_ms=round(ttfb or 0, 1),
            e2e_latency_ms=round(e2e, 1),
            audio_duration_s=round(audio_dur, 2),
            rtf=round(rtf, 4),
            success=True,
        )

    except Exception as e:
        torch.cuda.synchronize()
        e2e = (time.perf_counter() - t_start) * 1000
        return RequestResult(
            request_id=request_id,
            concurrency_level=concurrency_level,
            sentence=sentence,
            ttfb_ms=0,
            e2e_latency_ms=round(e2e, 1),
            audio_duration_s=0,
            rtf=float('inf'),
            success=False,
            error=str(e),
        )

def run_non_streaming_request(model, voice_prompt, request_id, concurrency_level):
    """Non-streaming request for batch queue simulation."""
    sentence = AGENT_SENTENCES[request_id % len(AGENT_SENTENCES)]

    try:
        torch.cuda.synchronize()
        t_start = time.perf_counter()

        wavs, sr = model.generate_voice_clone(
            text=sentence,
            language="English",
            voice_clone_prompt=voice_prompt,
            non_streaming_mode=True,
        )

        torch.cuda.synchronize()
        e2e = (time.perf_counter() - t_start) * 1000

        audio = wavs[0]
        audio_dur = len(audio) / sr
        rtf = (e2e / 1000) / audio_dur

        return RequestResult(
            request_id=request_id,
            concurrency_level=concurrency_level,
            sentence=sentence,
            ttfb_ms=round(e2e, 1),  # non-streaming, TTFB = E2E
            e2e_latency_ms=round(e2e, 1),
            audio_duration_s=round(audio_dur, 2),
            rtf=round(rtf, 4),
            success=True,
        )
    except Exception as e:
        torch.cuda.synchronize()
        e2e = (time.perf_counter() - t_start) * 1000
        return RequestResult(
            request_id=request_id,
            concurrency_level=concurrency_level,
            sentence=sentence,
            ttfb_ms=0, e2e_latency_ms=round(e2e, 1),
            audio_duration_s=0, rtf=float('inf'),
            success=False, error=str(e),
        )

def run_concurrency_test(model, voice_prompt, concurrency, num_requests, use_streaming=False):
    """Fire num_requests with given concurrency level."""
    results = []

    if use_streaming:
        # Streaming: sequential since GPU can't truly parallelize model inference
        # But we simulate concurrent arrivals with a thread pool
        with ThreadPoolExecutor(max_workers=concurrency) as pool:
            # Use a lock since GPU model isn't thread-safe for concurrent inference
            gpu_lock = threading.Lock()

            def locked_request(rid):
                with gpu_lock:
                    return run_single_request(model, voice_prompt, rid, concurrency)

            futures = {pool.submit(locked_request, i): i for i in range(num_requests)}
            for f in as_completed(futures):
                results.append(f.result())
    else:
        # Non-streaming sequential (simulates queue processing)
        for i in range(num_requests):
            r = run_non_streaming_request(model, voice_prompt, i, concurrency)
            results.append(r)

    return results

def compute_stats(results: List[RequestResult]):
    """Compute percentile stats from results."""
    successful = [r for r in results if r.success]
    if not successful:
        return {"error": "all requests failed"}

    ttfbs = [r.ttfb_ms for r in successful]
    e2es = [r.e2e_latency_ms for r in successful]
    rtfs = [r.rtf for r in successful]
    audio_durs = [r.audio_duration_s for r in successful]

    total_audio = sum(audio_durs)
    total_wall = sum(r.e2e_latency_ms for r in results) / 1000  # includes failures

    return {
        "num_requests": len(results),
        "num_success": len(successful),
        "num_failed": len(results) - len(successful),
        "ttfb_p50_ms": round(np.percentile(ttfbs, 50), 1),
        "ttfb_p90_ms": round(np.percentile(ttfbs, 90), 1),
        "ttfb_p95_ms": round(np.percentile(ttfbs, 95), 1),
        "ttfb_p99_ms": round(np.percentile(ttfbs, 99), 1),
        "ttfb_min_ms": round(min(ttfbs), 1),
        "ttfb_max_ms": round(max(ttfbs), 1),
        "e2e_p50_ms": round(np.percentile(e2es, 50), 1),
        "e2e_p90_ms": round(np.percentile(e2es, 90), 1),
        "e2e_p95_ms": round(np.percentile(e2es, 95), 1),
        "e2e_p99_ms": round(np.percentile(e2es, 99), 1),
        "e2e_min_ms": round(min(e2es), 1),
        "e2e_max_ms": round(max(e2es), 1),
        "rtf_avg": round(np.mean(rtfs), 4),
        "rtf_p50": round(np.percentile(rtfs, 50), 4),
        "rtf_p99": round(np.percentile(rtfs, 99), 4),
        "total_audio_s": round(total_audio, 1),
        "throughput_audio_s_per_s": round(total_audio / (total_wall) if total_wall > 0 else 0, 3),
    }

# ═══════════════════════════════════════════════════
print("=" * 70)
print("  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST")
print("  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation")
print("=" * 70)

# ─────────────────────────────────────────────
# Test 1: Streaming (TTFB measurement) — faster-qwen3-tts
# ─────────────────────────────────────────────
print("\n[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)")
print("  Measures real TTFB for voice agent responsiveness")

from faster_qwen3_tts import FasterQwen3TTS

fast_model = FasterQwen3TTS.from_pretrained(
    "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    device="cuda",
    dtype=torch.bfloat16,
    attn_implementation="sdpa",
)

# Warmup
print("  Warming up...")
for i in range(3):
    for _ in fast_model.generate_voice_clone_streaming(
        text="Warmup sentence number one.", language="English",
        ref_audio=REF_AUDIO, ref_text=REF_TEXT,
    ):
        pass

NUM_REQUESTS = 30
print(f"  Running {NUM_REQUESTS} streaming requests (sequential, measures true TTFB)...")

streaming_results = []
for i in range(NUM_REQUESTS):
    r = run_single_request(fast_model, None, i, 1)
    streaming_results.append(r)
    if (i + 1) % 10 == 0:
        print(f"    {i+1}/{NUM_REQUESTS} done")

streaming_stats = compute_stats(streaming_results)

print(f"\n  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):")
print(f"    TTFB p50:  {streaming_stats['ttfb_p50_ms']:>8.0f} ms")
print(f"    TTFB p90:  {streaming_stats['ttfb_p90_ms']:>8.0f} ms")
print(f"    TTFB p95:  {streaming_stats['ttfb_p95_ms']:>8.0f} ms")
print(f"    TTFB p99:  {streaming_stats['ttfb_p99_ms']:>8.0f} ms")
print(f"    E2E  p50:  {streaming_stats['e2e_p50_ms']:>8.0f} ms")
print(f"    E2E  p90:  {streaming_stats['e2e_p90_ms']:>8.0f} ms")
print(f"    RTF  avg:  {streaming_stats['rtf_avg']:>8.4f}")

del fast_model
torch.cuda.empty_cache()

# ─────────────────────────────────────────────
# Test 2: Batch throughput — how many requests/sec
# ─────────────────────────────────────────────
print("\n[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)")
print("  Simulates queued requests processed in batches")

from qwen_tts import Qwen3TTSModel

batch_model = Qwen3TTSModel.from_pretrained(
    "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    device_map="cuda:0",
    dtype=torch.bfloat16,
)
voice_prompt = batch_model.create_voice_clone_prompt(ref_audio=REF_AUDIO, ref_text=REF_TEXT)

# Test different batch sizes
BATCH_SIZES = [1, 2, 5, 10, 15, 20, 30]
batch_results = {}

print(f"\n  {'Batch':>6s}  {'GenTime':>8s}  {'Audio':>8s}  {'RTF':>8s}  {'Req/s':>8s}  {'AudioS/s':>8s}  {'Status'}")
print(f"  {'-'*6}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*8}  {'-'*10}")

for bs in BATCH_SIZES:
    sentences = [AGENT_SENTENCES[i % len(AGENT_SENTENCES)] for i in range(bs)]
    langs = ["English"] * bs

    try:
        # Warmup
        if bs <= 5:
            _ = batch_model.generate_voice_clone(
                text=sentences, language=langs,
                voice_clone_prompt=voice_prompt, non_streaming_mode=True,
            )

        # Timed run (3 runs, take avg)
        times = []
        audio_durs = []
        for run in range(3):
            torch.cuda.synchronize()
            t0 = time.perf_counter()
            wavs, sr = batch_model.generate_voice_clone(
                text=sentences, language=langs,
                voice_clone_prompt=voice_prompt, non_streaming_mode=True,
            )
            torch.cuda.synchronize()
            elapsed = time.perf_counter() - t0
            total_audio = sum(len(w) / sr for w in wavs)
            times.append(elapsed)
            audio_durs.append(total_audio)

        avg_time = np.mean(times)
        avg_audio = np.mean(audio_durs)
        rtf = avg_time / avg_audio
        rps = bs / avg_time
        audio_per_s = avg_audio / avg_time

        batch_results[bs] = {
            "batch_size": bs,
            "avg_gen_time_s": round(avg_time, 2),
            "avg_audio_s": round(avg_audio, 1),
            "rtf": round(rtf, 4),
            "requests_per_sec": round(rps, 2),
            "audio_seconds_per_sec": round(audio_per_s, 2),
        }

        print(f"  {bs:>6d}  {avg_time:>7.2f}s  {avg_audio:>7.1f}s  {rtf:>8.4f}  {rps:>7.2f}  {audio_per_s:>7.2f}  OK")

    except Exception as e:
        print(f"  {bs:>6d}  {'—':>8s}  {'—':>8s}  {'—':>8s}  {'—':>8s}  {'—':>8s}  FAIL: {e}")
        batch_results[bs] = {"batch_size": bs, "error": str(e)}

del batch_model, voice_prompt
torch.cuda.empty_cache()

# ─────────────────────────────────────────────
# Test 3: Sustained load simulation
# ─────────────────────────────────────────────
print("\n[TEST 3] SUSTAINED LOAD — 60s stress test (faster-qwen3-tts)")
print("  Simulates continuous voice agent traffic")

fast_model2 = FasterQwen3TTS.from_pretrained(
    "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    device="cuda",
    dtype=torch.bfloat16,
    attn_implementation="sdpa",
)

# Warmup
for i in range(3):
    chunks = list(fast_model2.generate_voice_clone_streaming(
        text="Warmup.", language="English",
        ref_audio=REF_AUDIO, ref_text=REF_TEXT,
    ))

DURATION_S = 60
print(f"  Running for {DURATION_S}s...")
sustained_results = []
t_start = time.time()
req_id = 0

while time.time() - t_start < DURATION_S:
    r = run_single_request(fast_model2, None, req_id, 1)
    sustained_results.append(r)
    req_id += 1

elapsed = time.time() - t_start
sustained_stats = compute_stats(sustained_results)

total_reqs = len(sustained_results)
total_audio_generated = sum(r.audio_duration_s for r in sustained_results if r.success)

print(f"\n  SUSTAINED LOAD RESULTS ({elapsed:.0f}s):")
print(f"    Total requests:      {total_reqs}")
print(f"    Requests/sec:        {total_reqs/elapsed:.2f}")
print(f"    Audio generated:     {total_audio_generated:.1f}s")
print(f"    Audio throughput:    {total_audio_generated/elapsed:.2f} sec audio/sec wall")
print(f"    TTFB p50:            {sustained_stats['ttfb_p50_ms']:.0f} ms")
print(f"    TTFB p95:            {sustained_stats['ttfb_p95_ms']:.0f} ms")
print(f"    TTFB p99:            {sustained_stats['ttfb_p99_ms']:.0f} ms")
print(f"    E2E  p50:            {sustained_stats['e2e_p50_ms']:.0f} ms")
print(f"    E2E  p95:            {sustained_stats['e2e_p95_ms']:.0f} ms")

del fast_model2
torch.cuda.empty_cache()

# ─────────────────────────────────────────────
# Capacity Planning
# ─────────────────────────────────────────────
# Find best batch throughput
best_batch = max(batch_results.values(), key=lambda x: x.get("requests_per_sec", 0))
best_rps = best_batch.get("requests_per_sec", 0)
best_audio_ps = best_batch.get("audio_seconds_per_sec", 0)

# Avg voice agent call: ~30s of TTS audio per 5-min call
# Each user generates roughly 6s of TTS audio per minute
AUDIO_PER_USER_PER_MIN = 6  # seconds of TTS per user per minute

max_concurrent_users_batch = int(best_audio_ps * 60 / AUDIO_PER_USER_PER_MIN) if AUDIO_PER_USER_PER_MIN > 0 else 0
max_concurrent_users_streaming = int(sustained_stats["throughput_audio_s_per_s"] * 60 / AUDIO_PER_USER_PER_MIN)

# Save everything
all_results = {
    "gpu": "NVIDIA A100-SXM4-80GB",
    "model": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
    "dtype": "bfloat16",
    "streaming_test": {
        "engine": "faster-qwen3-tts (CUDA graphs)",
        "num_requests": len(streaming_results),
        "stats": streaming_stats,
    },
    "batch_throughput": batch_results,
    "sustained_load": {
        "engine": "faster-qwen3-tts (CUDA graphs)",
        "duration_s": round(elapsed, 1),
        "total_requests": total_reqs,
        "stats": sustained_stats,
    },
    "capacity_planning": {
        "assumption_audio_per_user_per_min_s": AUDIO_PER_USER_PER_MIN,
        "max_concurrent_users_batch_mode": max_concurrent_users_batch,
        "max_concurrent_users_streaming_mode": max_concurrent_users_streaming,
        "best_batch_size": best_batch.get("batch_size"),
        "best_batch_rps": best_rps,
        "best_batch_audio_s_per_s": best_audio_ps,
    },
}

results_path = os.path.join(OUTPUT_DIR, "load_test_results.json")
with open(results_path, "w") as f:
    json.dump(all_results, f, indent=2)

print("\n" + "=" * 70)
print("  PRODUCTION CAPACITY — Qwen3-TTS-12Hz-0.6B-Base")
print("  Single NVIDIA A100-SXM4-80GB")
print("=" * 70)
print()
print(f"  STREAMING MODE (low latency, per-request):")
print(f"    Engine:          faster-qwen3-tts (CUDA graphs)")
print(f"    TTFB p50:        {streaming_stats['ttfb_p50_ms']:.0f} ms")
print(f"    TTFB p99:        {streaming_stats['ttfb_p99_ms']:.0f} ms")
print(f"    E2E  p50:        {streaming_stats['e2e_p50_ms']:.0f} ms")
print(f"    Throughput:      {sustained_stats['throughput_audio_s_per_s']:.2f} sec audio/sec")
print(f"    Max users:       ~{max_concurrent_users_streaming} concurrent")
print()
print(f"  BATCH MODE (max throughput, queued):")
print(f"    Engine:          qwen_tts (batch={best_batch.get('batch_size')})")
print(f"    Requests/sec:    {best_rps:.2f}")
print(f"    Throughput:      {best_audio_ps:.2f} sec audio/sec")
print(f"    Max users:       ~{max_concurrent_users_batch} concurrent")
print()
print(f"  FOR 10,000 USERS:")
if max_concurrent_users_batch > 0:
    gpus_needed_batch = int(np.ceil(10000 / max_concurrent_users_batch))
    gpus_needed_stream = int(np.ceil(10000 / max_concurrent_users_streaming))
    print(f"    Batch mode:      ~{gpus_needed_batch} x A100-80GB GPUs")
    print(f"    Streaming mode:  ~{gpus_needed_stream} x A100-80GB GPUs")
print()
print(f"  Results saved to: {results_path}")
print("=" * 70)

-    chunks = list(fast_model2.generate_voice_clone_streaming(

+    for _ in fast_model2.generate_voice_clone_streaming(

print(f"  Running for {DURATION_S}s...")


Bash

Run production load test with fixed streaming API


Output: `torch_dtype` is deprecated! Use `dtype` instead!

`torch_dtype` is deprecated! Use `dtype` instead!


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]


Output:
[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]


Output:
[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]


Output:
[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.

`torch_dtype` is deprecated! Use `dtype` instead!
======================================================================
  Qwen3-TTS-12Hz-0.6B-Base — PRODUCTION LOAD TEST
  GPU: NVIDIA A100-SXM4-80GB | English Voice Agent Simulation
======================================================================

[TEST 1] STREAMING — faster-qwen3-tts (CUDA graphs)
  Measures real TTFB for voice agent responsiveness

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 58867.42it/s]
  Warming up...
Warming up predictor (3 runs)...
Capturing CUDA graph for predictor...
CUDA graph captured!
Warming up talker graph (3 runs)...
Capturing CUDA graph for talker decode...
Talker CUDA graph captured!
  Running 30 streaming requests (sequential, measures true TTFB)...
    10/30 done
    20/30 done
    30/30 done

  STREAMING RESULTS (faster-qwen3-tts, CUDA graphs):
    TTFB p50:       378 ms
    TTFB p90:       380 ms
    TTFB p95:       380 ms
    TTFB p99:       380 ms
    E2E  p50:      1604 ms
    E2E  p90:      1978 ms
    RTF  avg:    0.3544

[TEST 2] BATCH THROUGHPUT — qwen_tts (max concurrency)
  Simulates queued requests processed in batches

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files: 100%|██████████| 4/4 [00:00<00:00, 24855.13it/s]
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.


Output:
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.
Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation.