Output:
Chunk ID: 8e3c47
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2070
Output:
    print(f"{'=' * 72}")

    print(f"  Success:     {len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"  Wall time:   {level.wall_time_seconds:.2f}s")
    print(f"  Throughput:  {level.throughput_rps:.2f} req/s")
    print(f"  Audio total: {level.audio_duration_total:.2f}s")
    print(f"  Eff. RTF:    {level.effective_rtf:.3f}")

    if level.observed_inflight_max() is not None:
        print(f"  Peak inflight: {level.observed_inflight_max()}")

    latency = level.latency_stats
    if latency:
        print("\n  Latency (ms):")
        print(
            f"    min={latency['min']:.0f} avg={latency['avg']:.0f} "
            f"p50={latency['p50']:.0f} p95={latency['p95']:.0f} p99={latency['p99']:.0f} max={latency['max']:.0f}"
        )

    ttfb = level.ttfb_stats
    if ttfb and not level.mode.startswith("stream"):
        print("\n  Server TTFB (ms):")
        print(
            f"    min={ttfb['min']:.0f} avg={ttfb['avg']:.0f} "
            f"p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}"
        )

    if level.mode == "non-streaming":
        llm = level.timing_stats("generation_ms")
        if llm:
            print("\n  Generation (ms):")
            print(f"    min={llm['min']:.0f} avg={llm['avg']:.0f} p50={llm['p50']:.0f} p95={llm['p95']:.0f} max={llm['max']:.0f}")
        wall = level.timing_stats("llm_batch_wall_ms_total")
        if wall:
            print("\n  LLM batch wall (ms):")
            print(f"    min={wall['min']:.0f} avg={wall['avg']:.0f} p50={wall['p50']:.0f} p95={wall['p95']:.0f} max={wall['max']:.0f}")
        gpu = level.timing_stats("llm_batch_gpu_ms_total")
        if gpu:
            print("\n  LLM batch GPU (ms):")
            print(f"    min={gpu['min']:.0f} avg={gpu['avg']:.0f} p50={gpu['p50']:.0f} p95={gpu['p95']:.0f} max={gpu['max']:.0f}")
        parse = level.timing_stats("llm_parse_ms")
        if parse:
            print("\n  Parse (ms):")
            print(f"    avg={parse['avg']:.0f} p95={parse['p95']:.0f}")
        decode = level.timing_stats("llm_decode_wall_ms_total")
        if decode:
            print("\n  Decode (ms):")
            print(f"    avg={decode['avg']:.0f} p95={decode['p95']:.0f}")
        bdecode = level.timing_stats("bicodec_decode_wall_ms")
        if bdecode:
            print("\n  BiCodec decode (ms):")
            print(f"    avg={bdecode['avg']:.0f} p95={bdecode['p95']:.0f}")
        bdecode_gpu = level.timing_stats("bicodec_decode_gpu_ms")
        if bdecode_gpu:
            print("\n  BiCodec decode GPU (ms):")
            print(f"    avg={bdecode_gpu['avg']:.0f} p95={bdecode_gpu['p95']:.0f}")

        token_ms, token_count = level.timing_rate("generation_ms", "llm_token_total")
        if token_count:
            print(f"  Token timing: avg {token_ms * 1000:.3f} us/token")

    if level.gpu_summary and "memory_mb" in level.gpu_summary:
        gpu_summary = level.gpu_summary
        mem = gpu_summary["memory_mb"]
        util = gpu_summary["gpu_util_pct"]
        temp = gpu_summary["temperature_c"]
        print("\n  GPU:")
        print(f"    Memory: {mem['avg']:.0f}MB avg, {mem['max']:.0f}MB peak / {mem['total']:.0f}MB")
        print(f"    Util:   {util['avg']:.0f}% avg, {util['max']:.0f}% peak")
        print(f"    Temp:   {temp['avg']:.0f}C avg, {temp['max']:.0f}C peak")

    errors = level.error_summary()
    if errors:
        print("\n  Errors:")
        for key, count in sorted(errors.items(), key=lambda kv: -kv[1]):
            print(f"    [{count}x] {key}")

def print_summary_table(levels: List[LevelResult]):
    print(f"\n{'=' * 110}")
    print("  STRESS BENCHMARK SUMMARY")
    print(f"{'=' * 110}")
    header = f"{'Level':<26} {'OK%':>5} {'RPS':>7} {'p50':>7} {'p95':>7} {'GPU%':>5} {'GPUmem':>8} {'RTF':>6} {'Inflight':>8}"
    print(header)
    print("-" * 110)

    for level in levels:
        lat = level.latency_stats
        p50 = f"{lat.get('p50', 0):.0f}" if lat else "---"
        p95 = f"{lat.get('p95', 0):.0f}" if lat else "---"
        gpu_util = level.gpu_summary.get("gpu_util_pct", {}).get("avg", 0)
        gpu_mem = level.gpu_summary.get("memory_mb", {}).get("max", 0)
        inflight = level.observed_inflight_max() or 0
        print(
            f"{level.level_name:<26} "
            f"{level.success_rate:>4.0%} "
            f"{level.throughput_rps:>7.2f} "
            f"{p50:>7} "
            f"{p95:>7} "
            f"{gpu_util:>4.0f}% "
            f"{gpu_mem:>7.0f}MB "
            f"{level.effective_rtf:>6.3f} "
            f"{inflight:>8d}"
        )

    print("=" * 110)

def save_results(levels: List[LevelResult], path: str):
    payload = []
    for level in levels:
        payload.append({
            "level": level.level_name,
            "concurrency": level.concurrency,
            "mode": level.mode,
            "chunking": level.chunking,
            "text_category": level.text_category,
            "success_rate": level.success_rate,
            "throughput_rps": level.throughput_rps,
            "wall_time_s": level.wall_time_seconds,
            "latency_ms": level.latency_stats,
            "ttfb_ms": level.ttfb_stats,
            "effective_rtf": level.effective_rtf,
            "audio_total_seconds": level.audio_duration_total,
            "observed_inflight_max": level.observed_inflight_max(),
            "errors": level.error_summary(),
            "gpu": level.gpu_summary,
            "timing_averages": {
                "generation_ms": level.timing_stats("generation_ms"),
                "llm_batch_wall_ms_total": level.timing_stats("llm_batch_wall_ms_total"),
                "llm_batch_gpu_ms_total": level.timing_stats("llm_batch_gpu_ms_total"),
                "llm_parse_ms": level.timing_stats("llm_parse_ms"),
                "llm_decode_wall_ms_total": level.timing_stats("llm_decode_wall_ms_total"),
                "bicodec_decode_wall_ms": level.timing_stats("bicodec_decode_wall_ms"),
                "bicodec_decode_gpu_ms": level.timing_stats("bicodec_decode_gpu_ms"),
            },
            "requests": [
                {
                    "request_id": r.request_id,
                    "success": r.success,
                    "status_code": r.status_code,
                    "latency_ms": r.latency_ms,
                    "ttfb_ms": r.ttfb_ms,
                    "server_ttfb_ms": r.server_ttfb_ms,
                    "audio_bytes": r.audio_bytes,
                    "audio_seconds": r.audio_seconds,
                    "request_inflight": r.request_inflight,
                    "error": r.error,
                    "timing": r.timing,
                }
                for r in level.results
            ],
        })

    with open(path, "w") as fd:
        json.dump(payload, fd, indent=2)

async def run_stress_test(args):
    base_url = args.url

    try:
        async with httpx.AsyncClient() as client:
            response = await client.get(f"{base_url}/v1/tts/health", timeout=10)
            health = response.json()
            print(f"  Health: {health.get('status')}")
            print(f"  Version: {health.get('model_version')}")
            print(f"  GPU: {health.get('gpu_available')}")
    except Exception as e:
        print(f"ERROR: server not reachable at {base_url}: {e}")
        sys.exit(1)

    levels = [int(v) for v in args.levels.split(",") if v.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    mode_label = "streaming" if args.stream else "non-streaming"
    results: List[LevelResult] = []

    print(f"\nWarming up (1 request, non-streaming)...")
    warmup = await run_level(
        base_url=base_url,
        num_requests=1,
        concurrency=1,
        text_category=args.text,
        chunking=args.chunking,
        max_tokens=args.max_tokens,
        request_timeout=args.timeout,
        stream=False,
        gpu_monitor=gpu_monitor,
    )
    print(f"  warmup ok: {warmup.success_rate:.0%}")

    for conc in levels:
        num_requests = max(conc * 3, 10)
        num_requests = min(num_requests, 200)
        level = await run_level(
            base_url=base_url,
            num_requests=num_requests,
            concurrency=conc,
            text_category=args.text,


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Inspecting related print and save logic**


sed -n '300,430p' scripts/stress_test_runtime_detailed.py


Output:
Chunk ID: c01eeb
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1250
Output:
    async def wrapped(i: int) -> RequestResult:
        async with semaphore:
            speaker = SPEAKERS[i % len(SPEAKERS)]
            return await run_one_request(
                text=text,
                speaker=speaker,
                max_tokens=max_tokens,
                timeout_s=timeout_s,
                chunking=chunking,
            )

    gpu_monitor.snapshots.clear()
    t0 = time.perf_counter()
    level.results = await asyncio.gather(*[wrapped(i) for i in range(total_requests)])
    level.wall_time_s = time.perf_counter() - t0
    level.gpu_summary = gpu_monitor.summary()
    return level

def print_level(level: LevelResult) -> None:
    print(f"\n=== concurrency={level.concurrency} requests={level.total_requests} ===")
    print(f"success={len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"wall={level.wall_time_s:.2f}s throughput={level.throughput_rps:.2f} req/s")
    lat = level.latency_stats()
    if lat:
        print(f"latency_ms avg={lat['avg']:.0f} p50={lat['p50']:.0f} p95={lat['p95']:.0f} max={lat['max']:.0f}")

    keys = [
        "generation_ms",
        "llm_batch_wall_ms_total",
        "llm_batch_gpu_ms_total",
        "llm_decode_wall_ms_total",
        "llm_decode_gpu_ms_total",
        "llm_parse_ms",
        "llm_time_per_token_ms",
        "bicodec_decode_wall_ms",
        "bicodec_decode_gpu_ms",
    ]
    for key in keys:
        st = level.timing_stats(key)
        if st:
            print(f"{key}: avg={st['avg']:.2f} p50={st['p50']:.2f} p95={st['p95']:.2f} max={st['max']:.2f}")

    if "memory_mb" in level.gpu_summary:
        g = level.gpu_summary
        print(
            "gpu: util_avg={:.0f}% util_max={:.0f}% mem_avg={:.0f}MB mem_max={:.0f}MB".format(
                g["gpu_util_pct"]["avg"],
                g["gpu_util_pct"]["max"],
                g["memory_mb"]["avg"],
                g["memory_mb"]["max"],
            )
        )

    errors: Dict[str, int] = {}
    for r in level.failures:
        k = r.error or "failed"
        errors[k] = errors.get(k, 0) + 1
    for k, v in sorted(errors.items(), key=lambda kv: -kv[1]):
        print(f"error[{v}] {k}")

def save_results(path: str, levels: List[LevelResult], args: argparse.Namespace) -> None:
    payload = {
        "config": {
            "levels": args.levels,
            "text": args.text,
            "max_tokens": args.max_tokens,
            "timeout_s": args.timeout,
            "chunking": args.chunking,
            "requests_multiplier": args.requests_multiplier,
            "timestamp": int(time.time()),
        },
        "levels": [],
    }
    for lvl in levels:
        payload["levels"].append(
            {
                "concurrency": lvl.concurrency,
                "total_requests": lvl.total_requests,
                "success_rate": lvl.success_rate,
                "throughput_rps": lvl.throughput_rps,
                "wall_time_s": lvl.wall_time_s,
                "latency_ms": lvl.latency_stats(),
                "gpu": lvl.gpu_summary,
                "timing": {
                    "generation_ms": lvl.timing_stats("generation_ms"),
                    "llm_batch_wall_ms_total": lvl.timing_stats("llm_batch_wall_ms_total"),
                    "llm_batch_gpu_ms_total": lvl.timing_stats("llm_batch_gpu_ms_total"),
                    "llm_decode_wall_ms_total": lvl.timing_stats("llm_decode_wall_ms_total"),
                    "llm_decode_gpu_ms_total": lvl.timing_stats("llm_decode_gpu_ms_total"),
                    "llm_parse_ms": lvl.timing_stats("llm_parse_ms"),
                    "llm_time_per_token_ms": lvl.timing_stats("llm_time_per_token_ms"),
                    "bicodec_decode_wall_ms": lvl.timing_stats("bicodec_decode_wall_ms"),
                    "bicodec_decode_gpu_ms": lvl.timing_stats("bicodec_decode_gpu_ms"),
                },
                "requests": [
                    {
                        "success": r.success,
                        "latency_ms": r.latency_ms,
                        "audio_bytes": r.audio_bytes,
                        "audio_seconds": r.audio_seconds,
                        "speaker": r.speaker,
                        "text_length": r.text_length,
                        "error": r.error,
                        "timing": r.timing,
                    }
                    for r in lvl.results
                ],
            }
        )
    with open(path, "w", encoding="utf-8") as fd:
        json.dump(payload, fd, indent=2)

async def main_async(args: argparse.Namespace) -> None:
    text = TEST_TEXTS[args.text]
    levels = [int(x.strip()) for x in args.levels.split(",") if x.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    # warmup
    await run_one_request(
        text=text,
        speaker=SPEAKERS[0],
        max_tokens=args.max_tokens,
        timeout_s=args.timeout,
        chunking=args.chunking,
    )

    results: List[LevelResult] = []


Summary: auto


**Adding timeline helpers and prints**


sed -n '540,620p' veena3modal/api/fastapi_app.py


Output:
Chunk ID: 023977
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1012
Output:
                    "X-TTFB-ms": str(metrics.get("ttfb_ms", 0)),
                    "X-RTF": f"{rtf:.3f}",
                    "X-Text-Chunked": str(metrics.get("text_chunked", False)).lower(),
                    "X-SR-Applied": str(metrics.get("sr_applied", False)).lower(),
                }
                headers.update(_build_perf_headers(metrics, request_inflight=inflight_current))

                if req.seed is not None:
                    headers["X-Seed"] = str(req.seed)

                # Fire-and-forget sentence storage (non-blocking, after generation)
                sentence_store = _get_sentence_store()
                sentence_store.store_fire_and_forget(
                    request_id=request_id,
                    text=text,
                    speaker=speaker,
                    stream=False,
                    format=req.format,
                    temperature=req.temperature,
                    top_k=req.top_k,
                    top_p=req.top_p,
                    max_tokens=req.max_tokens,
                    repetition_penalty=req.repetition_penalty,
                    seed=req.seed,
                    text_chunked=metrics.get("text_chunked", False),
                    ttfb_ms=metrics.get("ttfb_ms"),
                    audio_duration_seconds=audio_duration,
                )

                # Log successful completion (lifecycle event + metrics)
                total_duration_ms = int(total_time * 1000)
                log_request_completed(
                    request_id=request_id,
                    status_code=200,
                    total_duration_ms=total_duration_ms,
                    audio_duration_seconds=audio_duration,
                    rtf=rtf,
                    chunks_sent=1,
                )
                metrics_request_completed(
                    status_code=200,
                    duration_seconds=total_time,
                    speaker=speaker,
                    stream=False,
                )
                record_ttfb(ttfb_seconds=metrics.get("ttfb_ms", 0) / 1000, speaker=speaker, stream=False)
                record_rtf(rtf=rtf, speaker=speaker)
                record_audio_duration(duration_seconds=audio_duration, speaker=speaker)

                # High-cardinality details are kept in logs; summarize key timings.
                logger.info(
                    "request_perf_summary",
                    extra={
                        "request_id": request_id,
                        "speaker": speaker,
                        "generation_ms": metrics.get("generation_ms"),
                        "llm_token_total": metrics.get("llm_token_total"),
                        "llm_batch_count": metrics.get("llm_batch_count"),
                        "llm_batch_wall_ms_total": metrics.get("llm_batch_wall_ms_total"),
                        "llm_batch_gpu_ms_total": metrics.get("llm_batch_gpu_ms_total"),
                        "bicodec_decode_wall_ms": metrics.get("bicodec_decode_wall_ms"),
                        "bicodec_decode_gpu_ms": metrics.get("bicodec_decode_gpu_ms"),
                        "request_inflight": metrics.get("request_inflight"),
                    },
                )

                clear_request_context()

                # Apply format encoding if not WAV
                media_type = "audio/wav"
                if target_format != AudioFormat.WAV.value:
                    try:
                        from veena3modal.audio.encoder import (
                            OpusEncoder, MP3Encoder, MuLawEncoder, FLACEncoder
                        )

                        # Get sample rate from output (after potential SR)
                        sample_rate = metrics.get("output_sample_rate", 16000)

                        # Strip WAV header to get PCM data
                        pcm_data = audio_bytes[44:] if len(audio_bytes) > 44 else audio_bytes


**Adding timeline metrics and stage shares**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_optimized_local.py
@@
     header_map = {
+        "x-timeline-first-batch-ms": "timeline_llm_first_batch_ms",
+        "x-timeline-llm-done-ms": "timeline_llm_done_ms",
+        "x-timeline-bicodec-done-ms": "timeline_bicodec_done_ms",
+        "x-timeline-total-ms": "timeline_total_ms",
         "x-llm-token-total": "llm_token_total",
         "x-llm-batch-count": "llm_batch_count",
         "x-llm-batch-wall-ms": "llm_batch_wall_ms_total",
@@
         bdecode_gpu = level.timing_stats("bicodec_decode_gpu_ms")
         if bdecode_gpu:
             print("\n  BiCodec decode GPU (ms):")
             print(f"    avg={bdecode_gpu['avg']:.0f} p95={bdecode_gpu['p95']:.0f}")

+        tl_first = level.timing_stats("timeline_llm_first_batch_ms")
+        tl_llm_done = level.timing_stats("timeline_llm_done_ms")
+        tl_parse_done = level.timing_stats("timeline_parse_done_ms")
+        tl_bicodec_done = level.timing_stats("timeline_bicodec_done_ms")
+        tl_total = level.timing_stats("timeline_total_ms")
+        if tl_total:
+            print("\n  Timeline markers (ms):")
+            if tl_first:
+                print(f"    first_batch avg={tl_first['avg']:.0f} p95={tl_first['p95']:.0f}")
+            if tl_llm_done:
+                print(f"    llm_done avg={tl_llm_done['avg']:.0f} p95={tl_llm_done['p95']:.0f}")
+            if tl_parse_done:
+                print(f"    parse_done avg={tl_parse_done['avg']:.0f} p95={tl_parse_done['p95']:.0f}")
+            if tl_bicodec_done:
+                print(f"    bicodec_done avg={tl_bicodec_done['avg']:.0f} p95={tl_bicodec_done['p95']:.0f}")
+            print(f"    request_done avg={tl_total['avg']:.0f} p95={tl_total['p95']:.0f}")
+
+            llm_avg = wall.get("avg", 0.0) if wall else 0.0
+            bicodec_avg = bdecode.get("avg", 0.0) if bdecode else 0.0
+            total_avg = tl_total.get("avg", 0.0)
+            if total_avg > 0:
+                other_avg = max(0.0, total_avg - llm_avg - bicodec_avg)
+                print(
+                    "    Stage share: llm={:.1f}% bicodec={:.1f}% other={:.1f}%".format(
+                        (llm_avg / total_avg) * 100,
+                        (bicodec_avg / total_avg) * 100,
+                        (other_avg / total_avg) * 100,
+                    )
+                )
+
+            timeline_samples = [
+                r for r in level.successes
+                if isinstance(r.timing.get("timeline_total_ms"), (int, float))
+            ]
+            if timeline_samples:
+                sample = sorted(timeline_samples, key=lambda r: r.latency_ms)[len(timeline_samples) // 2]
+                print(
+                    "    Sample (median-latency): first_batch={:.1f} llm_done={:.1f} parse_done={:.1f} "
+                    "bicodec_done={:.1f} total={:.1f}".format(
+                        float(sample.timing.get("timeline_llm_first_batch_ms", 0.0)),
+                        float(sample.timing.get("timeline_llm_done_ms", 0.0)),
+                        float(sample.timing.get("timeline_parse_done_ms", 0.0)),
+                        float(sample.timing.get("timeline_bicodec_done_ms", 0.0)),
+                        float(sample.timing.get("timeline_total_ms", 0.0)),
+                    )
+                )
+
         token_ms, token_count = level.timing_rate("generation_ms", "llm_token_total")
         if token_count:
             print(f"  Token timing: avg {token_ms * 1000:.3f} us/token")
@@
                 "llm_parse_ms": level.timing_stats("llm_parse_ms"),
                 "llm_decode_wall_ms_total": level.timing_stats("llm_decode_wall_ms_total"),
                 "bicodec_decode_wall_ms": level.timing_stats("bicodec_decode_wall_ms"),
                 "bicodec_decode_gpu_ms": level.timing_stats("bicodec_decode_gpu_ms"),
+                "timeline_llm_first_batch_ms": level.timing_stats("timeline_llm_first_batch_ms"),
+                "timeline_llm_done_ms": level.timing_stats("timeline_llm_done_ms"),
+                "timeline_parse_done_ms": level.timing_stats("timeline_parse_done_ms"),
+                "timeline_bicodec_done_ms": level.timing_stats("timeline_bicodec_done_ms"),
+                "timeline_total_ms": level.timing_stats("timeline_total_ms"),
             },
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_optimized_local.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Extending patch to runtime script**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_runtime_detailed.py
@@
     keys = [
         "generation_ms",
+        "timeline_llm_first_batch_ms",
+        "timeline_llm_done_ms",
+        "timeline_parse_done_ms",
+        "timeline_bicodec_done_ms",
+        "timeline_total_ms",
         "llm_batch_wall_ms_total",
         "llm_batch_gpu_ms_total",
         "llm_decode_wall_ms_total",
         "llm_decode_gpu_ms_total",
         "llm_parse_ms",
@@
     for key in keys:
         st = level.timing_stats(key)
         if st:
             print(f"{key}: avg={st['avg']:.2f} p50={st['p50']:.2f} p95={st['p95']:.2f} max={st['max']:.2f}")
+
+    tl_total = level.timing_stats("timeline_total_ms")
+    llm_wall = level.timing_stats("llm_batch_wall_ms_total")
+    bicodec = level.timing_stats("bicodec_decode_wall_ms")
+    if tl_total and llm_wall:
+        total_avg = tl_total.get("avg", 0.0)
+        llm_avg = llm_wall.get("avg", 0.0)
+        bicodec_avg = bicodec.get("avg", 0.0) if bicodec else 0.0
+        other_avg = max(0.0, total_avg - llm_avg - bicodec_avg)
+        if total_avg > 0:
+            print(
+                "timeline_share: llm={:.1f}% bicodec={:.1f}% other={:.1f}%".format(
+                    (llm_avg / total_avg) * 100,
+                    (bicodec_avg / total_avg) * 100,
+                    (other_avg / total_avg) * 100,
+                )
+            )
+        samples = [
+            r for r in level.successes
+            if isinstance(r.timing.get("timeline_total_ms"), (int, float))
+        ]
+        if samples:
+            sample = sorted(samples, key=lambda r: r.latency_ms)[len(samples) // 2]
+            t = sample.timing
+            print(
+                "timeline_sample_ms: first_batch={:.1f} llm_done={:.1f} parse_done={:.1f} "
+                "bicodec_done={:.1f} total={:.1f}".format(
+                    float(t.get("timeline_llm_first_batch_ms", 0.0)),
+                    float(t.get("timeline_llm_done_ms", 0.0)),
+                    float(t.get("timeline_parse_done_ms", 0.0)),
+                    float(t.get("timeline_bicodec_done_ms", 0.0)),
+                    float(t.get("timeline_total_ms", 0.0)),
+                )
+            )

     if "memory_mb" in level.gpu_summary:
         g = level.gpu_summary
         print(
@@
                     "llm_decode_gpu_ms_total": lvl.timing_stats("llm_decode_gpu_ms_total"),
                     "llm_parse_ms": lvl.timing_stats("llm_parse_ms"),
                     "llm_time_per_token_ms": lvl.timing_stats("llm_time_per_token_ms"),
                     "bicodec_decode_wall_ms": lvl.timing_stats("bicodec_decode_wall_ms"),
                     "bicodec_decode_gpu_ms": lvl.timing_stats("bicodec_decode_gpu_ms"),
+                    "timeline_llm_first_batch_ms": lvl.timing_stats("timeline_llm_first_batch_ms"),
+                    "timeline_llm_done_ms": lvl.timing_stats("timeline_llm_done_ms"),
+                    "timeline_parse_done_ms": lvl.timing_stats("timeline_parse_done_ms"),
+                    "timeline_bicodec_done_ms": lvl.timing_stats("timeline_bicodec_done_ms"),
+                    "timeline_total_ms": lvl.timing_stats("timeline_total_ms"),
                 },
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_runtime_detailed.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Adding timeline to request summary**


Input:
*** Begin Patch
*** Update File: veena3modal/api/fastapi_app.py
@@
                         "llm_batch_count": metrics.get("llm_batch_count"),
                         "llm_batch_wall_ms_total": metrics.get("llm_batch_wall_ms_total"),
                         "llm_batch_gpu_ms_total": metrics.get("llm_batch_gpu_ms_total"),
+                        "timeline_llm_first_batch_ms": metrics.get("timeline_llm_first_batch_ms"),
+                        "timeline_llm_done_ms": metrics.get("timeline_llm_done_ms"),
+                        "timeline_bicodec_done_ms": metrics.get("timeline_bicodec_done_ms"),
+                        "timeline_total_ms": metrics.get("timeline_total_ms"),
                         "bicodec_decode_wall_ms": metrics.get("bicodec_decode_wall_ms"),
                         "bicodec_decode_gpu_ms": metrics.get("bicodec_decode_gpu_ms"),
                         "request_inflight": metrics.get("request_inflight"),
                     },
                 )
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/api/fastapi_app.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Integrating timeline markers into performance metrics**


I’ve wired the timeline markers through API headers/log payloads and both benchmark scripts now print timeline-stage stats plus stage-share percentages. I’m running a compile check, then I’ll rerun benchmarks to generate fresh timeline-based results.


python3 -m py_compile veena3modal/core/pipeline.py veena3modal/api/fastapi_app.py scripts/stress_test_optimized_local.py scripts/stress_test_runtime_detailed.py


rg -n "timeline_|x-timeline|timeline_markers|timeline_share|timeline_sample_ms" veena3modal/core/pipeline.py veena3modal/api/fastapi_app.py scripts/stress_test_optimized_local.py scripts/stress_test_runtime_detailed.py


Output:
Chunk ID: b7c716
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 0
Output:


Output:
Chunk ID: 938e5c
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 2417
Output:
scripts/stress_test_runtime_detailed.py:329:        "timeline_llm_first_batch_ms",
scripts/stress_test_runtime_detailed.py:330:        "timeline_llm_done_ms",
scripts/stress_test_runtime_detailed.py:331:        "timeline_parse_done_ms",
scripts/stress_test_runtime_detailed.py:332:        "timeline_bicodec_done_ms",
scripts/stress_test_runtime_detailed.py:333:        "timeline_total_ms",
scripts/stress_test_runtime_detailed.py:348:    tl_total = level.timing_stats("timeline_total_ms")
scripts/stress_test_runtime_detailed.py:358:                "timeline_share: llm={:.1f}% bicodec={:.1f}% other={:.1f}%".format(
scripts/stress_test_runtime_detailed.py:366:            if isinstance(r.timing.get("timeline_total_ms"), (int, float))
scripts/stress_test_runtime_detailed.py:372:                "timeline_sample_ms: first_batch={:.1f} llm_done={:.1f} parse_done={:.1f} "
scripts/stress_test_runtime_detailed.py:374:                    float(t.get("timeline_llm_first_batch_ms", 0.0)),
scripts/stress_test_runtime_detailed.py:375:                    float(t.get("timeline_llm_done_ms", 0.0)),
scripts/stress_test_runtime_detailed.py:376:                    float(t.get("timeline_parse_done_ms", 0.0)),
scripts/stress_test_runtime_detailed.py:377:                    float(t.get("timeline_bicodec_done_ms", 0.0)),
scripts/stress_test_runtime_detailed.py:378:                    float(t.get("timeline_total_ms", 0.0)),
scripts/stress_test_runtime_detailed.py:434:                    "timeline_llm_first_batch_ms": lvl.timing_stats("timeline_llm_first_batch_ms"),
scripts/stress_test_runtime_detailed.py:435:                    "timeline_llm_done_ms": lvl.timing_stats("timeline_llm_done_ms"),
scripts/stress_test_runtime_detailed.py:436:                    "timeline_parse_done_ms": lvl.timing_stats("timeline_parse_done_ms"),
scripts/stress_test_runtime_detailed.py:437:                    "timeline_bicodec_done_ms": lvl.timing_stats("timeline_bicodec_done_ms"),
scripts/stress_test_runtime_detailed.py:438:                    "timeline_total_ms": lvl.timing_stats("timeline_total_ms"),
veena3modal/api/fastapi_app.py:134:        "x-timeline-first-batch-ms": "timeline_llm_first_batch_ms",
veena3modal/api/fastapi_app.py:135:        "x-timeline-llm-done-ms": "timeline_llm_done_ms",
veena3modal/api/fastapi_app.py:136:        "x-timeline-bicodec-done-ms": "timeline_bicodec_done_ms",
veena3modal/api/fastapi_app.py:137:        "x-timeline-total-ms": "timeline_total_ms",
veena3modal/api/fastapi_app.py:191:        "timeline_markers",
veena3modal/api/fastapi_app.py:192:        "timeline_prompt_ready_ms",
veena3modal/api/fastapi_app.py:193:        "timeline_sampling_ready_ms",
veena3modal/api/fastapi_app.py:194:        "timeline_llm_first_batch_ms",
veena3modal/api/fastapi_app.py:195:        "timeline_llm_done_ms",
veena3modal/api/fastapi_app.py:196:        "timeline_parse_done_ms",
veena3modal/api/fastapi_app.py:197:        "timeline_validation_done_ms",
veena3modal/api/fastapi_app.py:198:        "timeline_bicodec_done_ms",
veena3modal/api/fastapi_app.py:199:        "timeline_wav_done_ms",
veena3modal/api/fastapi_app.py:200:        "timeline_request_done_ms",
veena3modal/api/fastapi_app.py:201:        "timeline_to_first_batch_ms",
veena3modal/api/fastapi_app.py:202:        "timeline_first_batch_to_llm_done_ms",
veena3modal/api/fastapi_app.py:203:        "timeline_post_llm_ms",
veena3modal/api/fastapi_app.py:204:        "timeline_parse_to_bicodec_done_ms",
veena3modal/api/fastapi_app.py:205:        "timeline_bicodec_to_request_done_ms",
veena3modal/api/fastapi_app.py:206:        "timeline_total_ms",
veena3modal/api/fastapi_app.py:600:                        "timeline_llm_first_batch_ms": metrics.get("timeline_llm_first_batch_ms"),
veena3modal/api/fastapi_app.py:601:                        "timeline_llm_done_ms": metrics.get("timeline_llm_done_ms"),
veena3modal/api/fastapi_app.py:602:                        "timeline_bicodec_done_ms": metrics.get("timeline_bicodec_done_ms"),
veena3modal/api/fastapi_app.py:603:                        "timeline_total_ms": metrics.get("timeline_total_ms"),
veena3modal/core/pipeline.py:153:        timeline_marks: List[Dict[str, float]] = [{"stage": "request_start", "t_ms": 0.0, "dt_ms": 0.0}]
veena3modal/core/pipeline.py:155:        perf["timeline_markers"] = timeline_marks
veena3modal/core/pipeline.py:156:        perf["timeline_request_start_ms"] = 0.0
veena3modal/core/pipeline.py:163:            timeline_marks.append(
veena3modal/core/pipeline.py:170:            perf[f"timeline_{stage}_ms"] = float(now_ms)
veena3modal/core/pipeline.py:171:            perf[f"timeline_{stage}_delta_ms"] = float(dt_ms)
veena3modal/core/pipeline.py:175:            if not timeline_marks or timeline_marks[-1].get("stage") != "request_done":
veena3modal/core/pipeline.py:179:            perf["timeline_markers"] = list(timeline_marks)
veena3modal/core/pipeline.py:180:            perf["timeline_total_ms"] = float(perf.get("timeline_request_done_ms", 0.0))
veena3modal/core/pipeline.py:182:            first_batch_ms = perf.get("timeline_llm_first_batch_ms")
veena3modal/core/pipeline.py:183:            llm_done_ms = perf.get("timeline_llm_done_ms")
veena3modal/core/pipeline.py:184:            parse_done_ms = perf.get("timeline_parse_done_ms")
veena3modal/core/pipeline.py:185:            bicodec_done_ms = perf.get("timeline_bicodec_done_ms")
veena3modal/core/pipeline.py:186:            request_done_ms = perf.get("timeline_request_done_ms")
veena3modal/core/pipeline.py:189:                perf["timeline_to_first_batch_ms"] = float(first_batch_ms)
veena3modal/core/pipeline.py:191:                perf["timeline_first_batch_to_llm_done_ms"] = float(llm_done_ms - first_batch_ms)
veena3modal/core/pipeline.py:193:                perf["timeline_post_llm_ms"] = float(request_done_ms - llm_done_ms)
veena3modal/core/pipeline.py:195:                perf["timeline_parse_to_bicodec_done_ms"] = float(bicodec_done_ms - parse_done_ms)
veena3modal/core/pipeline.py:197:                perf["timeline_bicodec_to_request_done_ms"] = float(request_done_ms - bicodec_done_ms)
veena3modal/core/pipeline.py:283:                first_batch_timeline_ms = _mark("llm_first_batch")
veena3modal/core/pipeline.py:290:                        "timeline_ms": round(first_batch_timeline_ms, 3),
veena3modal/core/pipeline.py:323:        llm_done_timeline_ms = _mark("llm_done")
veena3modal/core/pipeline.py:330:                "timeline_ms": round(llm_done_timeline_ms, 3),
veena3modal/core/pipeline.py:349:        parse_done_timeline_ms = _mark("parse_done")
veena3modal/core/pipeline.py:357:                "timeline_ms": round(parse_done_timeline_ms, 3),
veena3modal/core/pipeline.py:455:        bicodec_done_timeline_ms = _mark("bicodec_done")
veena3modal/core/pipeline.py:463:                "timeline_ms": round(bicodec_done_timeline_ms, 3),
veena3modal/core/pipeline.py:486:                "timeline_total_ms": round(float(perf.get("timeline_total_ms", 0.0)), 3),
scripts/stress_test_optimized_local.py:314:        "x-timeline-first-batch-ms": "timeline_llm_first_batch_ms",
scripts/stress_test_optimized_local.py:315:        "x-timeline-llm-done-ms": "timeline_llm_done_ms",
scripts/stress_test_optimized_local.py:316:        "x-timeline-bicodec-done-ms": "timeline_bicodec_done_ms",
scripts/stress_test_optimized_local.py:317:        "x-timeline-total-ms": "timeline_total_ms",
scripts/stress_test_optimized_local.py:681:        tl_first = level.timing_stats("timeline_llm_first_batch_ms")
scripts/stress_test_optimized_local.py:682:        tl_llm_done = level.timing_stats("timeline_llm_done_ms")
scripts/stress_test_optimized_local.py:683:        tl_parse_done = level.timing_stats("timeline_parse_done_ms")
scripts/stress_test_optimized_local.py:684:        tl_bicodec_done = level.timing_stats("timeline_bicodec_done_ms")
scripts/stress_test_optimized_local.py:685:        tl_total = level.timing_stats("timeline_total_ms")
scripts/stress_test_optimized_local.py:711:            timeline_samples = [
scripts/stress_test_optimized_local.py:713:                if isinstance(r.timing.get("timeline_total_ms"), (int, float))
scripts/stress_test_optimized_local.py:715:            if timeline_samples:
scripts/stress_test_optimized_local.py:716:                sample = sorted(timeline_samples, key=lambda r: r.latency_ms)[len(timeline_samples) // 2]
scripts/stress_test_optimized_local.py:720:                        float(sample.timing.get("timeline_llm_first_batch_ms", 0.0)),
scripts/stress_test_optimized_local.py:721:                        float(sample.timing.get("timeline_llm_done_ms", 0.0)),
scripts/stress_test_optimized_local.py:722:                        float(sample.timing.get("timeline_parse_done_ms", 0.0)),
scripts/stress_test_optimized_local.py:723:                        float(sample.timing.get("timeline_bicodec_done_ms", 0.0)),
scripts/stress_test_optimized_local.py:724:                        float(sample.timing.get("timeline_total_ms", 0.0)),
scripts/stress_test_optimized_local.py:806:                "timeline_llm_first_batch_ms": level.timing_stats("timeline_llm_first_batch_ms"),
scripts/stress_test_optimized_local.py:807:                "timeline_llm_done_ms": level.timing_stats("timeline_llm_done_ms"),
scripts/stress_test_optimized_local.py:808:                "timeline_parse_done_ms": level.timing_stats("timeline_parse_done_ms"),
scripts/stress_test_optimized_local.py:809:                "timeline_bicodec_done_ms": level.timing_stats("timeline_bicodec_done_ms"),
scripts/stress_test_optimized_local.py:810:                "timeline_total_ms": level.timing_stats("timeline_total_ms"),


**Running benchmarks with markers**


Instrumentation is compiled and wired end-to-end. I’m running fresh stress benchmarks now so the output includes marker timelines and stage-share breakdowns at each concurrency level.


python3 scripts/stress_test_runtime_detailed.py --levels 1,8,16,32 --text short --max-tokens 128 --timeout 120 --requests-multiplier 2 --min-requests 8 --gpu-memory 0.25 --output /tmp/stress_runtime_timeline.json


Output:
Chunk ID: 0e5d8b
Wall time: 1.0013 seconds
Process running with session ID 61637
Original token count: 0
Output:


**Implementing periodic polling**


Output:
Chunk ID: 8734e8
Wall time: 5.0013 seconds
Process running with session ID 61637
Original token count: 1277
Output:
[2026-02-17 08:30:36] INFO tts_runtime.py:154: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.25
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-17 08:30:37 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 08:30:37 [model.py:1561] Using max model len 4096
INFO 02-17 08:30:37 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 08:30:37 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:38 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}


**Planning periodic commentary**


Cold-start is in progress for this run; once model init finishes, the new timeline markers will appear in each level report (`first_batch`, `llm_done`, `bicodec_done`, `request_done`) with stage-share percentages.


Output:
Chunk ID: a4bdbe
Wall time: 5.0014 seconds
Process running with session ID 61637
Original token count: 511
Output:
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:40 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:54647 backend=nccl
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:40 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:41 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:41 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=3902659) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=3902659) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=3902659)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=3902659)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.24it/s]
(EngineCore_DP0 pid=3902659)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.24it/s]
(EngineCore_DP0 pid=3902659)
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:42 [default_loader.py:291] Loading weights took 0.87 seconds
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:43 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.406704 seconds
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:48 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:48 [backends.py:872] Dynamo bytecode transform time: 4.26 s


Output:
Chunk ID: 9909ec
Wall time: 5.0036 seconds
Process running with session ID 61637
Original token count: 727
Output:
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:51 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.635 s
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:51 [monitor.py:34] torch.compile takes 4.89 s in total
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:51 [gpu_worker.py:356] Available KV cache memory: 18.06 GiB
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:51 [kv_cache_utils.py:1307] GPU KV cache size: 1,577,712 tokens
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:51 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 385.18x
(EngineCore_DP0 pid=3902659)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 33.27it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 35.70it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 36.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 37.47it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 37.69it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 37.28it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 37.26it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 35.90it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:00<00:00, 35.84it/s]
(EngineCore_DP0 pid=3902659)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 29.69it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 33.93it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 35.29it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 35.92it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.93it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.28it/s]
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:54 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=3902659) INFO 02-17 08:30:54 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.70 seconds
[2026-02-17 08:30:55] INFO tts_runtime.py:162: Initializing prompt builder...
[2026-02-17 08:30:55] INFO tts_runtime.py:169: Initializing BiCodec decoder...


Model warmup is complete and requests are about to run; I’ll share the new timeline-based benchmark output once the concurrency levels finish.


**Polling process completion**


Output:
Chunk ID: c27e85
Wall time: 5.0012 seconds
Process running with session ID 61637
Original token count: 232
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)
[2026-02-17 08:31:04] INFO tts_runtime.py:176: Initializing TTS pipeline...
[2026-02-17 08:31:04] INFO tts_runtime.py:187: Initializing streaming pipeline...
[2026-02-17 08:31:04] INFO tts_runtime.py:237: ✅ TTS runtime initialized in 33184ms
[2026-02-17 08:31:04] INFO tts_runtime.py:238:    Model version: LLM
[2026-02-17 08:31:04] INFO tts_runtime.py:249: Speaker globals cache startup precompute skipped (disabled)
[2026-02-17 08:31:04] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:04] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:04] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:04] INFO pipeline.py:350: perf_token_parse_done


Summary: auto


Output:
Chunk ID: 069e91
Wall time: 5.0013 seconds
Process running with session ID 61637
Original token count: 7330
Output:
[2026-02-17 08:31:05] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:05] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:05] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:05] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:05] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:06] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:06] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:06] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:06] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:06] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:06] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:06] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:06] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:06] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:06] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:06] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:06] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:06] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:06] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:06] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:06] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:06] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:06] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:06] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:06] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:06] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:07] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:07] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:07] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:07] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:07] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:07] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:07] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:07] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:07] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:07] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:07] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:07] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:07] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:07] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:08] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:08] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:08] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:08] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:08] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:08] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:08] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:08] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:08] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:08] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:08] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:08] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:08] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:08] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:09] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:09] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:09] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:09] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:09] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:11] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:11] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:11] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:11] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:11] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:11] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:11] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:11] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:11] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:11] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:11] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:11] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:11] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:11] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:11] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:11] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:11] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:11] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:11] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:11] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:11] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:11] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:11] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:11] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:11] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:11] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:11] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:11] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:11] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:11] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:11] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:11] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:11] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:11] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:11] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:11] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:11] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:11] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:11] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:11] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:12] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:12] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:12] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:12] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:12] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:12] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:12] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:12] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:12] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:12] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:12] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:12] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:12] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:12] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:12] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:12] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:12] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:12] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:12] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:12] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:12] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:12] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:12] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:12] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:12] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:12] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:12] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:12] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:12] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:12] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:12] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:12] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:12] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:12] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:12] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:12] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:12] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:12] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:12] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:12] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:14] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:14] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:14] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:14] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:229: perf_non_stream_start
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:14] INFO pipeline.py:284: perf_first_llm_batch
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False
[2026-02-17 08:31:15] INFO pipeline.py:324: perf_llm_generation_done
[2026-02-17 08:31:15] INFO pipeline.py:350: perf_token_parse_done
[2026-02-17 08:31:15] INFO pipeline.py:456: perf_bicodec_decode_done
[2026-02-17 08:31:15] INFO pipeline.py:480: perf_non_stream_done
[2026-02-17 08:31:15] INFO tts_runtime.py:427: SR check: output_sample_rate='16khz', sr_service=False, is_loaded=False


**Waiting for command completion**