Output:
Chunk ID: fab3c2
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 4329
Output:
            "global_token_total": int(len(global_buffer)),
            "llm_parse_ms": float(sum(llm_parse_ms_samples)),
            "bicodec_decode_calls": int(len(bicodec_decode_wall_ms)),
            "llm_decode_calls": int(len(bicodec_decode_wall_ms)),
            "chunks_sent": int(total_audio_chunks),
            "audio_duration_seconds": float(total_samples_emitted_to_user / 16000.0),
            "ttfb_ms": float((t_first_chunk_emitted - t_request_start) * 1000.0 if t_first_chunk_emitted else 0.0),
            "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
            "stream_decode_interval_tokens": int(_STREAM_DECODE_INTERVAL),
            "stream_window_size_tokens": int(_STREAM_WINDOW_SIZE),
            "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
            "timeline_markers": list(timeline_marks),
            "timeline_total_ms": float((time.perf_counter() - t_request_start) * 1000.0),
        }

        decoder_batch_stats = {}
        if hasattr(self.snac_decoder, "get_batching_stats"):
            try:
                decoder_batch_stats = self.snac_decoder.get_batching_stats() or {}
            except Exception:
                decoder_batch_stats = {}
        if decoder_batch_stats:
            perf.update(decoder_batch_stats)

        self._apply_stats(perf, "llm_batch_wall_ms", llm_batch_wall_ms)
        self._apply_stats(perf, "llm_batch_gpu_ms", llm_batch_gpu_ms)
        self._apply_stats(perf, "llm_decode_wall_ms", llm_decode_wall_ms)
        self._apply_stats(perf, "llm_decode_gpu_ms", llm_decode_gpu_ms)
        self._apply_stats(perf, "tokens_per_batch", tokens_per_batch)
        self._apply_stats(perf, "llm_parse_step_ms", llm_parse_ms_samples)
        self._apply_stats(perf, "bicodec_decode_wall_ms", bicodec_decode_wall_ms)
        self._apply_stats(perf, "bicodec_decode_tokens", bicodec_decode_tokens)
        self._apply_stats(perf, "bicodec_decode_interval_ms", bicodec_decode_intervals_ms)

        perf["bicodec_decode_wall_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))
        perf["bicodec_decode_gpu_ms"] = 0.0
        perf["bicodec_decode_cpu_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))

        if total_generated_tokens > 0:
            perf["llm_time_per_token_ms"] = float(llm_generation_wall_ms) / float(total_generated_tokens)
        if llm_batch_wall_ms:
            perf["llm_time_per_batch_wall_ms"] = float(sum(llm_batch_wall_ms)) / float(len(llm_batch_wall_ms))
        if llm_batch_gpu_ms:
            perf["llm_time_per_batch_gpu_ms"] = float(sum(llm_batch_gpu_ms)) / float(len(llm_batch_gpu_ms))

        for mark in timeline_marks:
            stage = mark.get("stage")
            if not stage:
                continue
            perf[f"timeline_{stage}_ms"] = float(mark.get("t_ms", 0.0))
            perf[f"timeline_{stage}_delta_ms"] = float(mark.get("dt_ms", 0.0))

        tl_first_batch = perf.get("timeline_llm_first_batch_ms")
        tl_first_emit = perf.get("timeline_first_chunk_emitted_ms")
        tl_done = perf.get("timeline_request_done_ms")
        if isinstance(tl_first_batch, (int, float)):
            perf["timeline_to_first_batch_ms"] = float(tl_first_batch)
        if isinstance(tl_first_emit, (int, float)):
            perf["timeline_to_first_chunk_emitted_ms"] = float(tl_first_emit)
        if isinstance(tl_first_batch, (int, float)) and isinstance(tl_first_emit, (int, float)) and tl_first_emit >= tl_first_batch:
            perf["timeline_first_batch_to_first_chunk_emitted_ms"] = float(tl_first_emit - tl_first_batch)
        if isinstance(tl_first_emit, (int, float)) and isinstance(tl_done, (int, float)) and tl_done >= tl_first_emit:
            perf["timeline_first_chunk_to_request_done_ms"] = float(tl_done - tl_first_emit)

        sec_fields = {
            "time_in_queue": "llm_time_in_queue_ms",
            "scheduler_time": "llm_scheduler_ms",
            "model_forward_time": "llm_model_forward_ms",
            "model_execute_time": "llm_model_execute_ms",
            "first_token_latency": "llm_first_token_ms",
        }
        for src, dst in sec_fields.items():
            if src in final_request_metrics:
                perf[dst] = float(final_request_metrics[src]) * 1000.0

        def _delta_ms(start_s: Optional[float], end_s: Optional[float]) -> Optional[float]:
            if start_s is None or end_s is None:
                return None
            if end_s < start_s:
                return None
            return (end_s - start_s) * 1000.0

        queued_ts = final_request_metrics.get("queued_ts")
        scheduled_ts = final_request_metrics.get("scheduled_ts")
        first_token_ts = final_request_metrics.get("first_token_ts")
        last_token_ts = final_request_metrics.get("last_token_ts")
        q_to_sched = _delta_ms(queued_ts, scheduled_ts)
        sched_to_first = _delta_ms(scheduled_ts, first_token_ts)
        first_to_last = _delta_ms(first_token_ts, last_token_ts)
        queued_to_last = _delta_ms(queued_ts, last_token_ts)
        if q_to_sched is not None:
            perf["llm_queued_to_scheduled_ms"] = q_to_sched
            perf["llm_time_in_queue_ms"] = q_to_sched
        if sched_to_first is not None:
            perf["llm_scheduled_to_first_token_ms"] = sched_to_first
            perf["llm_scheduler_ms"] = sched_to_first
        if first_to_last is not None:
            perf["llm_first_to_last_token_ms"] = first_to_last
            perf["llm_model_execute_ms"] = first_to_last
        if queued_to_last is not None:
            perf["llm_queued_to_last_token_ms"] = queued_to_last
            perf["llm_request_lifecycle_ms"] = queued_to_last

        logger.debug(
            "streaming complete audio=%.2fs ttfb=%.0fms chunks=%d llm_tokens=%d",
            perf["audio_duration_seconds"],
            perf["ttfb_ms"],
            perf["chunks_sent"],
            perf["llm_token_total"],
        )

        if emit_progress:
            yield (b"", perf)

    async def generate_speech_stream_indic_first_chunk(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[tuple, None]:
        """
        Generate speech for the FIRST chunk of multi-chunk text, capturing global tokens.

        This method is specifically for chunked generation: it yields (audio_bytes, global_ids)
        tuples instead of just audio_bytes. The caller captures global_ids from the first
        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.

        Use Case (chunked streaming with voice consistency):
            globals_captured = None
            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
                if globals_captured is None and global_ids:
                    globals_captured = global_ids  # Capture once
                yield audio_bytes
            # Now use globals_captured for subsequent chunks

        Args:
            speaker: Speaker name
            text: First text chunk to synthesize
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max tokens to generate
            repetition_penalty: Prevent repetition
            seed: Random seed for reproducibility

        Yields:
            Tuple of (audio_bytes, global_ids)
            - audio_bytes: Raw PCM audio (int16, 16kHz)
            - global_ids: List of 32 global token IDs (populated after first decode, else empty)

        Thread Safety:
            This method is stateless and thread-safe. Each call creates its own
            request-scoped state. No global state is modified or shared.
        """
        import time
        t_start = time.time()

        # Build prompt using Indic prompt builder
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,
            seed=seed,
            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
        )

        # BiCodec token buffers
        semantic_buffer = []
        global_buffer = []
        processed_token_count = 0  # Used for cumulative mode; ignored for delta mode.
        total_audio_chunks = 0

        # BiCodec streaming configuration
        EXPECTED_GLOBAL_COUNT = 32
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
        CROSSFADE_MS = 50

        # Generate unique request ID
        import uuid
        request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None
        captured_globals: List[int] = []  # Will be populated once we have 32 globals

        # Stream tokens with BiCodec extraction
        async for request_output in results_generator:
            generated_ids = request_output.outputs[0].token_ids

            # In DELTA mode token_ids are already incremental.
            if _STREAM_USE_DELTA_OUTPUT:
                new_token_ids = generated_ids
            else:
                new_token_ids = generated_ids[processed_token_count:]
                processed_token_count = len(generated_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)

            # Capture global tokens once we have all 32
            if not captured_globals and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                captured_globals = global_buffer[:EXPECTED_GLOBAL_COUNT]

            # Process if we have all global tokens
            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                semantic_count = len(semantic_buffer)
                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
                        last_decode_count = semantic_count

                        all_semantic = semantic_buffer
                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]

                        if self.snac_decoder.enable_batching:
                            audio_bytes = await self.snac_decoder.decode_single_async(
                                semantic_ids=all_semantic,
                                global_ids=window_global,
                                trim_warmup=False,
                                use_sliding_window=False
                            )
                        else:
                            audio_bytes = self.snac_decoder.decode_streaming(
                                semantic_ids=all_semantic,
                                global_ids=window_global,
                                use_sliding_window=False,
                                trim_warmup=False
                            )

                        if audio_bytes:
                            total_samples_decoded = len(audio_bytes) // 2

                            if total_samples_decoded > total_samples_emitted_to_user:
                                new_bytes_start = total_samples_emitted_to_user * 2
                                new_audio_bytes = audio_bytes[new_bytes_start:]

                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                    previous_chunk_tail,
                                    new_audio_bytes,
                                    sample_rate_hz=16000,
                                    crossfade_ms=CROSSFADE_MS,
                                )

                                samples_emitted_in_this_chunk = len(to_emit) // 2
                                total_samples_emitted_to_user += samples_emitted_in_this_chunk

                                if to_emit:
                                    total_audio_chunks += 1
                                    # Yield tuple: (audio_bytes, global_ids)
                                    yield (to_emit, captured_globals)

        # Final chunk processing
        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,
                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
                use_sliding_window=False,
                trim_warmup=False
            )
            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    samples_emitted_in_final = len(to_emit) // 2
                    total_samples_emitted_to_user += samples_emitted_in_final

                    if to_emit:
                        total_audio_chunks += 1
                        yield (to_emit, captured_globals)

        # Flush remaining tail
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            yield (previous_chunk_tail, captured_globals)

        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000
        logger.debug(
            "first-chunk complete audio=%.2fs chunks=%d captured_globals=%d",
            audio_duration_s,
            total_audio_chunks,
            len(captured_globals),
        )

    async def generate_speech_stream_indic_continuation(
        self,
        speaker: str,
        text: str,
        global_ids: List[int],
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        emit_progress: bool = False,
    ) -> AsyncGenerator[Union[bytes, Tuple[bytes, Dict[str, Any]]], None]:
        """
        Generate speech for CONTINUATION chunks using pre-captured global tokens.

        This is the key method for voice consistency in chunked generation:
        - Uses global_ids from the first chunk to maintain identical voice
        - Model generates only semantic tokens (skips global token generation)
        - Ensures no voice drift across chunks

        CRITICAL for production:
        - global_ids is request-scoped, passed explicitly (no shared state)
        - Thread-safe: each request has its own state
        - No global caching that could cause cross-request contamination

        Args:
            speaker: Speaker name (must match first chunk)
            text: Continuation text chunk to synthesize
            global_ids: 32 global token IDs captured from first chunk
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max tokens to generate
            repetition_penalty: Prevent repetition
            seed: Random seed (use same as first chunk for consistency)

        Yields:
            Audio bytes (int16 PCM, 16kHz mono)

        Raises:
            ValueError: If global_ids doesn't contain exactly 32 tokens


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Planning adaptive decode intervals**


sed -n '1,260p' scripts/stress_test_streaming_runtime.py


sed -n '260,620p' scripts/stress_test_streaming_runtime.py


Output:
Chunk ID: dde0a1
Wall time: 0.0506 seconds
Process exited with code 0
Original token count: 2112
Output:
#!/usr/bin/env python3
"""
Direct-runtime streaming stress benchmark for Veena3 TTS.

Focuses on streaming KPIs:
- TTFB (time to first chunk)
- End-to-end latency
- Chunk cadence (inter-chunk interval)
- Emission speed (audio seconds emitted per second after first chunk)
- Throughput under concurrency
"""

from __future__ import annotations

import argparse
import asyncio
import json
import os
import statistics
import subprocess
import sys
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional

# Path setup (same pattern as local_server.py)
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
EXTERNAL_DIR = os.path.join(REPO_ROOT, "external")
for path in [os.path.join(EXTERNAL_DIR, "sparktts"), os.path.join(EXTERNAL_DIR, "AP-BWE"), REPO_ROOT]:
    if path not in sys.path:
        sys.path.insert(0, path)

os.environ["AUTH_BYPASS_MODE"] = "true"

TEST_TEXTS = {
    "short": "Hello, this is a quick streaming benchmark for voice agents.",
    "medium": (
        "This streaming benchmark sentence is designed to evaluate first audio latency, "
        "chunk cadence, and sustained emission speed under concurrent request load."
    ),
    "long": (
        "In a bustling city, a narrator collected voice notes from people walking through markets, "
        "subways, and parks. The stream needed to start quickly, stay stable, and deliver clear speech "
        "without stalls as many listeners connected at once."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float

class GPUMonitor:
    def __init__(self, interval_seconds: float = 0.5):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self) -> None:
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self) -> None:
        self._running = False
        if self._thread:
            self._thread.join(timeout=3)

    def _poll_loop(self) -> None:
        while self._running:
            try:
                res = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True,
                    text=True,
                    timeout=5,
                )
                if res.returncode == 0 and res.stdout:
                    parts = [p.strip() for p in res.stdout.strip().split(",")]
                    if len(parts) >= 3:
                        self.snapshots.append(
                            GPUSnapshot(
                                timestamp=time.time(),
                                memory_used_mb=float(parts[0]),
                                memory_total_mb=float(parts[1]),
                                gpu_utilization_pct=float(parts[2]),
                            )
                        )
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {"samples": 0, "error": "no GPU samples captured"}
        mem = [s.memory_used_mb for s in self.snapshots]
        util = [s.gpu_utilization_pct for s in self.snapshots]
        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem),
                "avg": statistics.mean(mem),
                "max": max(mem),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(util),
                "avg": statistics.mean(util),
                "max": max(util),
            },
        }

@dataclass
class RequestResult:
    success: bool
    latency_ms: float
    ttfb_ms: float
    chunks_sent: int
    total_bytes: int
    audio_seconds: float
    speaker: str
    error: Optional[str] = None
    inter_chunk_ms: List[float] = field(default_factory=list)
    tail_emit_speed_x: float = 0.0
    final_metrics: Dict[str, Any] = field(default_factory=dict)

@dataclass
class LevelResult:
    concurrency: int
    total_requests: int
    results: List[RequestResult] = field(default_factory=list)
    wall_time_s: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0.0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_s if self.wall_time_s > 0 else 0.0

    def _pct(self, vals: List[float], p: int) -> float:
        if not vals:
            return 0.0
        s = sorted(vals)
        idx = int((len(s) - 1) * (p / 100))
        return s[idx]

    def _stats(self, vals: List[float]) -> Dict[str, float]:
        if not vals:
            return {}
        return {
            "avg": statistics.mean(vals),
            "p50": self._pct(vals, 50),
            "p95": self._pct(vals, 95),
            "max": max(vals),
        }

    def latency_stats(self) -> Dict[str, float]:
        return self._stats([r.latency_ms for r in self.successes])

    def ttfb_stats(self) -> Dict[str, float]:
        return self._stats([r.ttfb_ms for r in self.successes if r.ttfb_ms > 0])

    def chunk_count_stats(self) -> Dict[str, float]:
        return self._stats([float(r.chunks_sent) for r in self.successes])

    def emit_speed_stats(self) -> Dict[str, float]:
        return self._stats([r.tail_emit_speed_x for r in self.successes if r.tail_emit_speed_x > 0])

    def inter_chunk_stats(self) -> Dict[str, float]:
        vals: List[float] = []
        for r in self.successes:
            vals.extend(r.inter_chunk_ms)
        return self._stats(vals)

    def timing_stats(self, key: str) -> Dict[str, float]:
        vals: List[float] = []
        for r in self.successes:
            v = r.final_metrics.get(key)
            if isinstance(v, (int, float)):
                vals.append(float(v))
        return self._stats(vals)

    def latency_over_metric_stats(self, key: str) -> Dict[str, float]:
        vals: List[float] = []
        for r in self.successes:
            base = r.final_metrics.get(key)
            if isinstance(base, (int, float)):
                vals.append(max(0.0, float(r.latency_ms) - float(base)))
        return self._stats(vals)

def init_runtime(
    model_path: str,
    gpu_mem: float,
    stream_output_kind: str,
    num_engines: int,
    max_num_batched_tokens: Optional[int],
    max_num_seqs: Optional[int],
    enable_chunked_prefill: Optional[bool],
    enable_prefix_caching: Optional[bool],
    disable_log_stats: Optional[bool],
    enforce_eager: Optional[bool],
) -> None:
    from veena3modal.local_server import resolve_model_paths
    from veena3modal.services.tts_runtime import initialize_runtime, is_initialized

    os.environ["VEENA3_STREAM_OUTPUT_KIND"] = stream_output_kind

    llm_path, bicodec_path = resolve_model_paths(model_path)
    if not is_initialized():
        initialize_runtime(
            model_path=llm_path,
            bicodec_path=bicodec_path,
            device="cuda",
            gpu_memory_utilization=gpu_mem,
            num_engines=num_engines,
            max_num_batched_tokens=max_num_batched_tokens,
            max_num_seqs=max_num_seqs,
            enable_chunked_prefill=enable_chunked_prefill,
            enable_prefix_caching=enable_prefix_caching,
            disable_log_stats=disable_log_stats,
            enforce_eager=enforce_eager,
            precompute_speaker_globals=False,
        )

async def run_one_request(
    text: str,
    speaker: str,
    max_tokens: int,
    timeout_s: float,
    enable_chunking: bool,
) -> RequestResult:
    from veena3modal.services import tts_runtime


Output:
Chunk ID: 05cdd4
Wall time: 0.0516 seconds
Process exited with code 0
Original token count: 3966
Output:
    from veena3modal.services import tts_runtime

    t0 = time.perf_counter()
    first_chunk_t: Optional[float] = None
    prev_chunk_t: Optional[float] = None
    inter_chunk_ms: List[float] = []
    chunks = 0
    total_bytes = 0
    final_metrics: Dict[str, Any] = {}

    async def _consume_stream() -> None:
        nonlocal first_chunk_t, prev_chunk_t, chunks, total_bytes, final_metrics
        gen = tts_runtime.generate_speech_streaming(
            text=text,
            speaker=speaker,
            max_tokens=max_tokens,
            enable_chunking=enable_chunking,
        )
        async for audio_chunk, metrics in gen:
            now = time.perf_counter()
            if first_chunk_t is None:
                first_chunk_t = now
            if prev_chunk_t is not None:
                inter_chunk_ms.append((now - prev_chunk_t) * 1000.0)
            prev_chunk_t = now
            chunks += 1
            total_bytes += len(audio_chunk)
            if isinstance(metrics, dict):
                # Keep reference; runtime may mutate this dict after the final chunk
                # when internal stream generation completes.
                final_metrics = metrics

    try:
        await asyncio.wait_for(_consume_stream(), timeout=timeout_s)
        end = time.perf_counter()
        latency_ms = (end - t0) * 1000.0
        ttfb_ms = ((first_chunk_t - t0) * 1000.0) if first_chunk_t is not None else 0.0
        audio_seconds = float(final_metrics.get("audio_duration_seconds", 0.0) or 0.0)
        tail_s = max(1e-6, (latency_ms - ttfb_ms) / 1000.0)
        tail_emit_speed_x = audio_seconds / tail_s if audio_seconds > 0 else 0.0
        return RequestResult(
            success=True,
            latency_ms=latency_ms,
            ttfb_ms=ttfb_ms,
            chunks_sent=chunks,
            total_bytes=total_bytes,
            audio_seconds=audio_seconds,
            speaker=speaker,
            inter_chunk_ms=inter_chunk_ms,
            tail_emit_speed_x=tail_emit_speed_x,
            final_metrics=dict(final_metrics),
        )
    except asyncio.TimeoutError:
        return RequestResult(
            success=False,
            latency_ms=(time.perf_counter() - t0) * 1000.0,
            ttfb_ms=0.0,
            chunks_sent=chunks,
            total_bytes=total_bytes,
            audio_seconds=0.0,
            speaker=speaker,
            error="timeout",
            inter_chunk_ms=inter_chunk_ms,
            final_metrics=dict(final_metrics),
        )
    except Exception as exc:
        return RequestResult(
            success=False,
            latency_ms=(time.perf_counter() - t0) * 1000.0,
            ttfb_ms=0.0,
            chunks_sent=chunks,
            total_bytes=total_bytes,
            audio_seconds=0.0,
            speaker=speaker,
            error=str(exc)[:200],
            inter_chunk_ms=inter_chunk_ms,
            final_metrics=dict(final_metrics),
        )

async def run_level(
    concurrency: int,
    total_requests: int,
    text: str,
    max_tokens: int,
    timeout_s: float,
    enable_chunking: bool,
    gpu_monitor: GPUMonitor,
) -> LevelResult:
    level = LevelResult(concurrency=concurrency, total_requests=total_requests)
    semaphore = asyncio.Semaphore(concurrency)

    async def wrapped(i: int) -> RequestResult:
        async with semaphore:
            speaker = SPEAKERS[i % len(SPEAKERS)]
            return await run_one_request(
                text=text,
                speaker=speaker,
                max_tokens=max_tokens,
                timeout_s=timeout_s,
                enable_chunking=enable_chunking,
            )

    gpu_monitor.snapshots.clear()
    t0 = time.perf_counter()
    level.results = await asyncio.gather(*[wrapped(i) for i in range(total_requests)])
    level.wall_time_s = time.perf_counter() - t0
    level.gpu_summary = gpu_monitor.summary()
    return level

def print_level(level: LevelResult) -> None:
    print(f"\n=== streaming concurrency={level.concurrency} requests={level.total_requests} ===")
    print(f"success={len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"wall={level.wall_time_s:.2f}s throughput={level.throughput_rps:.2f} req/s")
    lat = level.latency_stats()
    if lat:
        print(f"latency_ms avg={lat['avg']:.0f} p50={lat['p50']:.0f} p95={lat['p95']:.0f} max={lat['max']:.0f}")
    ttfb = level.ttfb_stats()
    if ttfb:
        print(f"ttfb_ms avg={ttfb['avg']:.0f} p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}")
    cc = level.chunk_count_stats()
    if cc:
        print(f"chunks_per_req avg={cc['avg']:.1f} p50={cc['p50']:.1f} p95={cc['p95']:.1f} max={cc['max']:.1f}")
    ic = level.inter_chunk_stats()
    if ic:
        print(
            "inter_chunk_ms avg={:.1f} p50={:.1f} p95={:.1f} max={:.1f}".format(
                ic["avg"], ic["p50"], ic["p95"], ic["max"]
            )
        )
    es = level.emit_speed_stats()
    if es:
        print(
            "tail_emit_speed_x avg={:.2f} p50={:.2f} p95={:.2f} max={:.2f}".format(
                es["avg"], es["p50"], es["p95"], es["max"]
            )
        )

    timing_keys = [
        "timeline_to_first_batch_ms",
        "timeline_to_first_chunk_emitted_ms",
        "timeline_total_ms",
        "llm_time_in_queue_ms",
        "llm_scheduler_ms",
        "llm_first_token_ms",
        "llm_request_lifecycle_ms",
        "llm_generation_wall_ms",
        "llm_time_per_token_ms",
        "llm_batch_wall_ms_total",
        "llm_decode_wall_ms_total",
        "tokens_per_batch_p50",
        "bicodec_decode_calls",
        "bicodec_decode_wall_ms_total",
        "bicodec_decode_interval_ms_p50",
        "batch_avg_size",
        "batch_max_seen",
    ]
    for key in timing_keys:
        st = level.timing_stats(key)
        if st:
            print(f"{key}: avg={st['avg']:.2f} p50={st['p50']:.2f} p95={st['p95']:.2f} max={st['max']:.2f}")

    latency_over_timeline = level.latency_over_metric_stats("timeline_total_ms")
    if latency_over_timeline:
        print(
            "latency_over_timeline_ms: avg={:.2f} p50={:.2f} p95={:.2f} max={:.2f}".format(
                latency_over_timeline["avg"],
                latency_over_timeline["p50"],
                latency_over_timeline["p95"],
                latency_over_timeline["max"],
            )
        )

    tl_total = level.timing_stats("timeline_total_ms")
    llm_total = level.timing_stats("llm_generation_wall_ms")
    bicodec_total = level.timing_stats("bicodec_decode_wall_ms_total")
    if tl_total and llm_total:
        total_avg = tl_total.get("avg", 0.0)
        llm_avg = llm_total.get("avg", 0.0)
        bicodec_avg = bicodec_total.get("avg", 0.0) if bicodec_total else 0.0
        other_avg = max(0.0, total_avg - llm_avg - bicodec_avg)
        if total_avg > 0:
            print(
                "timeline_share: llm={:.1f}% bicodec={:.1f}% other={:.1f}%".format(
                    (llm_avg / total_avg) * 100,
                    (bicodec_avg / total_avg) * 100,
                    (other_avg / total_avg) * 100,
                )
            )

    if "memory_mb" in level.gpu_summary:
        g = level.gpu_summary
        print(
            "gpu: util_avg={:.0f}% util_max={:.0f}% mem_avg={:.0f}MB mem_max={:.0f}MB".format(
                g["gpu_util_pct"]["avg"],
                g["gpu_util_pct"]["max"],
                g["memory_mb"]["avg"],
                g["memory_mb"]["max"],
            )
        )

def save_results(path: str, levels: List[LevelResult], args: argparse.Namespace) -> None:
    payload = {
        "config": {
            "levels": args.levels,
            "text": args.text,
            "max_tokens": args.max_tokens,
            "timeout_s": args.timeout,
            "requests_multiplier": args.requests_multiplier,
            "min_requests": args.min_requests,
            "gpu_memory": args.gpu_memory,
            "stream_output_kind": args.stream_output_kind,
            "chunking": args.chunking,
            "num_engines": args.num_engines,
            "max_num_batched_tokens": args.max_num_batched_tokens,
            "max_num_seqs": args.max_num_seqs,
            "enable_chunked_prefill": args.enable_chunked_prefill,
            "disable_chunked_prefill": args.disable_chunked_prefill,
            "disable_prefix_caching": args.disable_prefix_caching,
            "disable_engine_stats_logs": args.disable_engine_stats_logs,
            "enforce_eager": args.enforce_eager,
            "stream_decode_interval": args.stream_decode_interval,
            "stream_window_size": args.stream_window_size,
            "stream_min_semantic_first": args.stream_min_semantic_first,
            "stream_crossfade_ms": args.stream_crossfade_ms,
            "disable_windowed_decode": args.disable_windowed_decode,
            "disable_bicodec_batching": args.disable_bicodec_batching,
            "bicodec_batch_max": args.bicodec_batch_max,
            "bicodec_batch_timeout_ms": args.bicodec_batch_timeout_ms,
            "timestamp": int(time.time()),
        },
        "levels": [],
    }
    for lvl in levels:
        payload["levels"].append(
            {
                "concurrency": lvl.concurrency,
                "total_requests": lvl.total_requests,
                "success_rate": lvl.success_rate,
                "throughput_rps": lvl.throughput_rps,
                "wall_time_s": lvl.wall_time_s,
                "latency_ms": lvl.latency_stats(),
                "ttfb_ms": lvl.ttfb_stats(),
                "chunks_per_req": lvl.chunk_count_stats(),
                "inter_chunk_ms": lvl.inter_chunk_stats(),
                "tail_emit_speed_x": lvl.emit_speed_stats(),
                "gpu": lvl.gpu_summary,
                "timing": {
                    "timeline_to_first_batch_ms": lvl.timing_stats("timeline_to_first_batch_ms"),
                    "timeline_to_first_chunk_emitted_ms": lvl.timing_stats("timeline_to_first_chunk_emitted_ms"),
                    "timeline_total_ms": lvl.timing_stats("timeline_total_ms"),
                    "llm_time_in_queue_ms": lvl.timing_stats("llm_time_in_queue_ms"),
                    "llm_scheduler_ms": lvl.timing_stats("llm_scheduler_ms"),
                    "llm_first_token_ms": lvl.timing_stats("llm_first_token_ms"),
                    "llm_request_lifecycle_ms": lvl.timing_stats("llm_request_lifecycle_ms"),
                    "llm_generation_wall_ms": lvl.timing_stats("llm_generation_wall_ms"),
                    "llm_time_per_token_ms": lvl.timing_stats("llm_time_per_token_ms"),
                    "llm_batch_wall_ms_total": lvl.timing_stats("llm_batch_wall_ms_total"),
                    "llm_decode_wall_ms_total": lvl.timing_stats("llm_decode_wall_ms_total"),
                    "tokens_per_batch_p50": lvl.timing_stats("tokens_per_batch_p50"),
                    "bicodec_decode_calls": lvl.timing_stats("bicodec_decode_calls"),
                    "bicodec_decode_wall_ms_total": lvl.timing_stats("bicodec_decode_wall_ms_total"),
                    "bicodec_decode_interval_ms_p50": lvl.timing_stats("bicodec_decode_interval_ms_p50"),
                    "batch_avg_size": lvl.timing_stats("batch_avg_size"),
                    "batch_max_seen": lvl.timing_stats("batch_max_seen"),
                    "latency_over_timeline_ms": lvl.latency_over_metric_stats("timeline_total_ms"),
                },
                "requests": [
                    {
                        "success": r.success,
                        "latency_ms": r.latency_ms,
                        "ttfb_ms": r.ttfb_ms,
                        "chunks_sent": r.chunks_sent,
                        "total_bytes": r.total_bytes,
                        "audio_seconds": r.audio_seconds,
                        "speaker": r.speaker,
                        "error": r.error,
                        "inter_chunk_ms": r.inter_chunk_ms,
                        "tail_emit_speed_x": r.tail_emit_speed_x,
                        "final_metrics": r.final_metrics,
                    }
                    for r in lvl.results
                ],
            }
        )
    with open(path, "w", encoding="utf-8") as fd:
        json.dump(payload, fd, indent=2)

async def main_async(args: argparse.Namespace) -> None:
    text = TEST_TEXTS[args.text]
    levels = [int(x.strip()) for x in args.levels.split(",") if x.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    # warmup
    await run_one_request(
        text=text,
        speaker=SPEAKERS[0],
        max_tokens=args.max_tokens,
        timeout_s=args.timeout,
        enable_chunking=args.chunking,
    )

    results: List[LevelResult] = []
    for idx, c in enumerate(levels):
        n = max(args.min_requests, c * args.requests_multiplier)
        level = await run_level(
            concurrency=c,
            total_requests=n,
            text=text,
            max_tokens=args.max_tokens,
            timeout_s=args.timeout,
            enable_chunking=args.chunking,
            gpu_monitor=gpu_monitor,
        )
        print_level(level)
        results.append(level)
        if idx < len(levels) - 1:
            await asyncio.sleep(2)

    gpu_monitor.stop()
    save_results(args.output, results, args)
    print(f"\nresults saved: {args.output}")

def main() -> None:
    parser = argparse.ArgumentParser(description="Direct runtime streaming stress benchmark")
    parser.add_argument("--levels", default="16,32,64", help="Comma-separated concurrency levels")
    parser.add_argument("--text", choices=["short", "medium", "long"], default="short")
    parser.add_argument("--max-tokens", type=int, default=512, help="max_tokens sent to streaming generation")
    parser.add_argument("--timeout", type=float, default=180.0, help="Per-request timeout seconds")
    parser.add_argument("--requests-multiplier", type=int, default=2, help="Requests per level ~= concurrency * multiplier")
    parser.add_argument("--min-requests", type=int, default=8)
    parser.add_argument("--chunking", action="store_true", default=False, help="Enable long-text chunking path")
    parser.add_argument("--gpu-memory", type=float, default=0.25)
    parser.add_argument("--stream-output-kind", choices=["delta", "cumulative"], default="delta")
    parser.add_argument("--num-engines", type=int, default=1, help="Number of vLLM engines")
    parser.add_argument("--max-num-batched-tokens", type=int, default=None)
    parser.add_argument("--max-num-seqs", type=int, default=None)
    parser.add_argument("--enable-chunked-prefill", action="store_true", default=None)
    parser.add_argument("--disable-chunked-prefill", action="store_true", default=False)
    parser.add_argument("--disable-prefix-caching", action="store_true", default=False)
    parser.add_argument("--disable-engine-stats-logs", action="store_true", default=False)
    parser.add_argument("--enforce-eager", action="store_true", default=False)
    parser.add_argument("--stream-decode-interval", type=int, default=48)
    parser.add_argument("--stream-window-size", type=int, default=128)
    parser.add_argument("--stream-min-semantic-first", type=int, default=10)
    parser.add_argument("--stream-crossfade-ms", type=int, default=50)
    parser.add_argument("--disable-windowed-decode", action="store_true", default=False)
    parser.add_argument("--disable-bicodec-batching", action="store_true", default=False)
    parser.add_argument("--bicodec-batch-max", type=int, default=16)
    parser.add_argument("--bicodec-batch-timeout-ms", type=float, default=6.0)
    parser.add_argument("--model-path", type=str, default="")
    parser.add_argument("--output", default="stress_streaming_runtime.json")
    args = parser.parse_args()

    from veena3modal.local_server import DEFAULT_LOCAL_MODEL_DIR


Summary: auto


sed -n '620,860p' scripts/stress_test_streaming_runtime.py


Output:
Chunk ID: 0d39fc
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 391
Output:

    model_path = args.model_path or DEFAULT_LOCAL_MODEL_DIR

    os.environ["VEENA3_STREAM_OUTPUT_KIND"] = args.stream_output_kind
    os.environ["VEENA3_STREAM_DECODE_INTERVAL"] = str(args.stream_decode_interval)
    os.environ["VEENA3_STREAM_WINDOW_SIZE"] = str(args.stream_window_size)
    os.environ["VEENA3_STREAM_MIN_SEMANTIC_FIRST"] = str(args.stream_min_semantic_first)
    os.environ["VEENA3_STREAM_CROSSFADE_MS"] = str(args.stream_crossfade_ms)
    os.environ["VEENA3_STREAM_WINDOWED_DECODE"] = "0" if args.disable_windowed_decode else "1"
    os.environ["VEENA3_BICODEC_BATCHING"] = "0" if args.disable_bicodec_batching else "1"
    os.environ["VEENA3_BICODEC_BATCH_MAX"] = str(args.bicodec_batch_max)
    os.environ["VEENA3_BICODEC_BATCH_TIMEOUT_MS"] = str(args.bicodec_batch_timeout_ms)

    enable_chunked_prefill = args.enable_chunked_prefill
    if args.disable_chunked_prefill:
        enable_chunked_prefill = False

    init_runtime(
        model_path=model_path,
        gpu_mem=args.gpu_memory,
        stream_output_kind=args.stream_output_kind,
        num_engines=args.num_engines,
        max_num_batched_tokens=args.max_num_batched_tokens,
        max_num_seqs=args.max_num_seqs,
        enable_chunked_prefill=enable_chunked_prefill,
        enable_prefix_caching=(False if args.disable_prefix_caching else None),
        disable_log_stats=(True if args.disable_engine_stats_logs else None),
        enforce_eager=(True if args.enforce_eager else None),
    )
    asyncio.run(main_async(args))

if __name__ == "__main__":
    main()


**Planning batch tuning experiments**


I confirmed the benchmark harness already exposes the key scheduling knobs (`decode_interval`, `window_size`, BiCodec batch `max/timeout`), so I’ll run a focused sweep at high concurrency to find the best throughput/TTFB trade-off before proposing any deeper scheduler changes.


**Planning sequential GPU parameter sweeps**


set -euo pipefail
configs=(
  "i32_t6_b16_w128 32 128 16 6"
  "i64_t6_b16_w128 64 128 16 6"
  "i48_t4_b16_w128 48 128 16 4"
  "i48_t8_b16_w128 48 128 16 8"
  "i48_t8_b24_w128 48 128 24 8"
  "i48_t6_b16_w96 48 96 16 6"
)
for c in "${configs[@]}"; do
  read -r name interval window bmax btimeout <<<"$c"
  out="/tmp/stream_cfg_${name}.json"
  log="/tmp/stream_cfg_${name}.log"
  echo "=== RUN $name ==="
  python3 scripts/stress_test_streaming_runtime.py \
    --levels 96 \
    --text short \
    --max-tokens 1024 \
    --timeout 240 \
    --requests-multiplier 1 \
    --min-requests 8 \
    --gpu-memory 0.25 \
    --stream-output-kind delta \
    --stream-decode-interval "$interval" \
    --stream-window-size "$window" \
    --bicodec-batch-max "$bmax" \
    --bicodec-batch-timeout-ms "$btimeout" \
    --output "$out" >"$log" 2>&1
  python3 - <<PY
import json
p='$out'
d=json.load(open(p))
r=d['levels'][0]
t=r['timing']
print(f"{p}: thr={r['throughput_rps']:.2f} lat={r['latency_ms']['avg']:.1f} ttfb={r['ttfb_ms']['avg']:.1f} llm_tok_ms={t['llm_time_per_token_ms']['avg']:.2f} bicodec_total={t['bicodec_decode_wall_ms_total']['avg']:.1f} batch_avg={t['batch_avg_size']['avg']:.2f} util={r['gpu']['gpu_util_pct']['avg']:.1f}")
PY
done


Output:
Chunk ID: e54070
Wall time: 1.0005 seconds
Process running with session ID 98717
Original token count: 7
Output:
=== RUN i32_t6_b16_w128 ===


**Planning periodic commentary and polling**


I started a 6-run high-concurrency sweep (`c=96`) across decode interval, BiCodec batch timeout/max, and window size. I’ll report a ranked table as each run completes so we can choose the best throughput/latency point.


**Polling session initiated**


Output:
Chunk ID: 8e8cfd
Wall time: 5.0012 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: b32901
Wall time: 5.0011 seconds
Process running with session ID 98717
Original token count: 0
Output:


**Scheduling periodic updates**


The sweep is still in the first run (model load + warmup stage). Once it prints the first metric line, I’ll keep comparing deltas against the current `48/128/16/6` baseline.


**Implementing periodic polling**


Output:
Chunk ID: 466d0f
Wall time: 5.0009 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: 37c2b7
Wall time: 5.0007 seconds
Process running with session ID 98717
Original token count: 0
Output:


**Scheduling repeated poll after delay**


Output:
Chunk ID: 97ceeb
Wall time: 5.0002 seconds
Process running with session ID 98717
Original token count: 41
Output:
/tmp/stream_cfg_i32_t6_b16_w128.json: thr=23.09 lat=3723.0 ttfb=922.2 llm_tok_ms=16.23 bicodec_total=3149.9 batch_avg=11.40 util=68.5
=== RUN i64_t6_b16_w128 ===


**Reporting first run results**


First sweep result is in: `decode_interval=32` hurt throughput at `c=96` (`23.09 req/s`) and increased decode load, so smaller intervals are not the right direction for peak throughput. Continuing remaining configs now.


Output:
Chunk ID: 297974
Wall time: 5.0007 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: e5a05c
Wall time: 5.0016 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: 93f0a8
Wall time: 5.0022 seconds
Process running with session ID 98717
Original token count: 0
Output:


**Waiting for update trigger**


Second run (`decode_interval=64`) is still executing; once it completes I’ll check if the larger decode stride improves throughput without pushing TTFB too far.


Output:
Chunk ID: e134c5
Wall time: 5.0012 seconds
Process running with session ID 98717
Original token count: 0
Output:


Summary: auto


Output:
Chunk ID: 25ec8e
Wall time: 5.0015 seconds
Process running with session ID 98717
Original token count: 41
Output:
/tmp/stream_cfg_i64_t6_b16_w128.json: thr=26.19 lat=3205.1 ttfb=1196.5 llm_tok_ms=13.71 bicodec_total=1601.2 batch_avg=10.67 util=66.1
=== RUN i48_t4_b16_w128 ===


**Measuring improved throughput and high latency**


`decode_interval=64` improved peak throughput to `26.19 req/s` (best so far) with lower overall decode time, but TTFB rose to about `1.20s`. I’m now running timeout tuning to see if we can recover first-audio latency while holding throughput.


**Continuing polling process**


Output:
Chunk ID: b611a1
Wall time: 5.0007 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: b8b00d
Wall time: 5.0010 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: 87f2bb
Wall time: 5.0011 seconds
Process running with session ID 98717
Original token count: 0
Output:


**Preparing status update**


I have one clear lever now: larger decode intervals increase throughput by reducing decode pressure, but they tax first-chunk latency. I’m finishing the batch-timeout/max sweep to locate the best compromise.


Output:
Chunk ID: 397a01
Wall time: 5.0012 seconds
Process running with session ID 98717
Original token count: 41
Output:
/tmp/stream_cfg_i48_t4_b16_w128.json: thr=21.44 lat=3676.1 ttfb=1071.1 llm_tok_ms=16.00 bicodec_total=2906.7 batch_avg=7.44 util=66.3
=== RUN i48_t8_b16_w128 ===


**Continuing after timeout failure**


Lowering batch timeout to `4ms` reduced realized batch size and dropped throughput (`21.44 req/s`), so we should avoid aggressive timeout cuts at high concurrency. Continuing with `8ms` and larger batch-cap runs.


Output:
Chunk ID: accaff
Wall time: 5.0002 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: 7e81b6
Wall time: 5.0011 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: a1cfc0
Wall time: 5.0012 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: 676e20
Wall time: 5.0007 seconds
Process running with session ID 98717
Original token count: 0
Output:


No new metric line yet; this run is still in progress. After these two timeout/batch-cap tests, I’ll rank all configs against both `throughput` and `TTFB` so we can pick a production profile and an aggressive profile.


Output:
Chunk ID: f3dde7
Wall time: 5.0012 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: 2cb28b
Wall time: 5.0006 seconds
Process running with session ID 98717
Original token count: 41
Output:
/tmp/stream_cfg_i48_t8_b16_w128.json: thr=21.65 lat=3610.2 ttfb=998.0 llm_tok_ms=15.51 bicodec_total=2227.5 batch_avg=10.60 util=63.0
=== RUN i48_t8_b24_w128 ===


**Noting throughput issues persist**


`timeout=8ms` with `batch_max=16` still underperformed (`21.65 req/s`), even though batch size increased. I’m checking if raising `batch_max` to `24` unlocks better decode coalescing or just adds wait overhead.


Output:
Chunk ID: d2cec1
Wall time: 5.0019 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: 907818
Wall time: 5.0009 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: bc6410
Wall time: 5.0014 seconds
Process running with session ID 98717
Original token count: 0
Output:


**Considering periodic updates**


Current pattern is clear: overly long decode-batch wait hurts end-to-end flow even if batch size looks larger. I’ll finish the last two runs and then consolidate with an apples-to-apples summary table.


Output:
Chunk ID: dc338d
Wall time: 5.0011 seconds
Process running with session ID 98717
Original token count: 41
Output:
/tmp/stream_cfg_i48_t8_b24_w128.json: thr=23.12 lat=3543.9 ttfb=965.8 llm_tok_ms=15.14 bicodec_total=2063.6 batch_avg=15.87 util=69.1
=== RUN i48_t6_b16_w96 ===


Increasing `batch_max` to `24` recovered some throughput (`23.12 req/s`) but it’s still below the current best (`26.19 req/s` with interval 64). I’m running the final window-size test now.


Output:
Chunk ID: 17195c
Wall time: 5.0006 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: 035443
Wall time: 5.0005 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: 9cf01a
Wall time: 5.0009 seconds
Process running with session ID 98717
Original token count: 0
Output:


**Tracking update interval**


Final run is still processing; after this I’ll produce a compact Pareto view (max throughput, best TTFB, and balanced profile) and then implement the next scheduler change directly in code.


Output:
Chunk ID: 5c16cd
Wall time: 5.0006 seconds
Process running with session ID 98717
Original token count: 0
Output:


Output:
Chunk ID: 6b9acd
Wall time: 0.3647 seconds
Process exited with code 0
Original token count: 34
Output:
/tmp/stream_cfg_i48_t6_b16_w96.json: thr=23.21 lat=3510.1 ttfb=1071.5 llm_tok_ms=15.16 bicodec_total=2592.4 batch_avg=9.86 util=61.9


**Planning adaptive decode interval**


Output:
Chunk ID: f60cac
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 276
Output:
692:async def generate_speech_streaming(
751:    first_chunk_time = None
767:            if first_chunk_time is None:
768:                first_chunk_time = time.time()
769:                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
783:        # use continuation mode to skip the ~110ms global token pre-roll.
789:            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
803:            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
834:            if first_chunk_time is None:
835:                first_chunk_time = time.time()
836:                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
883:            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
910:                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
929:                # Use continuation method with cached globals
930:                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(


Summary: auto


sed -n '730,860p' veena3modal/services/tts_runtime.py


Output:
Chunk ID: 9d2f49
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1261
Output:
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()

    if runtime.streaming_pipeline is None:
        raise RuntimeError("Streaming pipeline not available")

    # Import audio utils for WAV header
    from veena3modal.audio.utils import create_wav_header

    # Metrics tracking
    metrics = {
        "ttfb_ms": 0,
        "chunks_sent": 0,
        "total_bytes": 0,
        "audio_duration_seconds": 0.0,
        "text_chunked": False,
    }

    start_time = time.time()
    first_chunk_time = None
    total_pcm_bytes = 0
    sample_rate = 16000  # BiCodec sample rate

    # Check if we need text chunking
    long_processor = _get_long_text_processor(runtime)
    needs_chunking = enable_chunking and long_processor.should_chunk(text)

    if needs_chunking:
        # Chunked streaming with voice consistency (global token caching)
        metrics["text_chunked"] = True
        async for audio_chunk, chunk_metrics in _stream_chunked_text(
            runtime, long_processor, text, speaker, temperature, top_k, top_p,
            max_tokens, repetition_penalty, seed, sample_rate
        ):
            # First chunk: prepend WAV header
            if first_chunk_time is None:
                first_chunk_time = time.time()
                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                # Create streaming WAV header (size=0 for unknown length)
                wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
                audio_chunk = wav_header + audio_chunk

            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
            metrics["chunks_sent"] += 1
            metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
            metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)

            yield audio_chunk, metrics
    else:
        # Simple streaming (no text chunking)
        # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
        # use continuation mode to skip the ~110ms global token pre-roll.
        # The model jumps straight to semantic token generation.
        cached_globals = runtime.speaker_global_cache.get(speaker)

        if cached_globals:
            # Fast path: use cached globals, skip global token generation entirely
            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                speaker=speaker,
                text=text,
                global_ids=cached_globals,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
                emit_progress=True,
            )
        else:
            # Fallback: no cached globals, generate them inline (original path)
            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
                speaker=speaker,
                text=text,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
                emit_progress=True,
            )

        async for stream_item in stream_gen:
            stream_metrics: Dict[str, Any] = {}
            audio_chunk = stream_item
            if (
                isinstance(stream_item, tuple)
                and len(stream_item) == 2
                and isinstance(stream_item[1], dict)
            ):
                audio_chunk = stream_item[0]
                stream_metrics = stream_item[1]

            if stream_metrics:
                metrics.update(stream_metrics)

            if not audio_chunk:
                # Metrics-only event from streaming pipeline.
                continue

            # First chunk: prepend WAV header
            if first_chunk_time is None:
                first_chunk_time = time.time()
                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
                audio_chunk = wav_header + audio_chunk

            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
            metrics["chunks_sent"] += 1
            metrics["total_bytes"] = total_pcm_bytes + 44
            metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)

            yield audio_chunk, metrics

async def _stream_chunked_text(
    runtime: TTSRuntime,
    long_processor,
    text: str,
    speaker: str,
    temperature: float,
    top_k: int,
    top_p: float,
    max_tokens: int,
    repetition_penalty: float,
    seed: Optional[int],
    sample_rate: int,
) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:


**Planning detailed queue wait instrumentation**


I’m going to add one more concrete optimization now: load-adaptive decode scheduling (fast first chunk, larger interval under pressure) plus queue-wait instrumentation inside the BiCodec batch worker so we can see decode queueing vs compute directly.


sed -n '1,260p' veena3modal/core/bicodec_decoder.py


Output:
Chunk ID: eb224f
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2293
Output:
"""
Spark TTS BiCodec Decoder

Decodes BiCodec audio tokens (semantic + global) to audio waveforms.
Replaces SNAC decoder for Spark TTS model.
"""

import asyncio
import logging
import os
import sys
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, List, Optional, Tuple

import numpy as np
import torch

# Ensure `sparktts` (vendored under external/) is importable in local/dev runs.
# In Modal, PYTHONPATH already includes `/root/external/sparktts`.
try:
    from sparktts.models.audio_tokenizer import BiCodecTokenizer
except ImportError:  # pragma: no cover - environment-dependent
    _this_file = Path(__file__).resolve()
    _sparktts_path = None
    for _p in [_this_file.parent] + list(_this_file.parents):
        _candidate = _p / "external" / "sparktts"
        if _candidate.is_dir():
            _sparktts_path = str(_candidate)
            break
    if _sparktts_path and _sparktts_path not in sys.path:
        sys.path.insert(0, _sparktts_path)
    from sparktts.models.audio_tokenizer import BiCodecTokenizer

from veena3modal.core.constants import (
    BICODEC_TOKENIZER_PATH,
    AUDIO_SAMPLE_RATE,
)

logger = logging.getLogger(__name__)

def _env_flag(name: str, default: bool) -> bool:
    raw = os.environ.get(name, "")
    if not raw:
        return default
    return raw.strip().lower() in {"1", "true", "yes", "on"}

def _env_int(name: str, default: int, minimum: int = 1) -> int:
    raw = os.environ.get(name, "")
    if not raw:
        return default
    try:
        value = int(raw)
    except ValueError:
        return default
    return max(minimum, value)

def _env_float(name: str, default: float, minimum: float = 0.0) -> float:
    raw = os.environ.get(name, "")
    if not raw:
        return default
    try:
        value = float(raw)
    except ValueError:
        return default
    return max(minimum, value)

@dataclass
class _BatchDecodeRequest:
    semantic_ids: List[int]
    global_ids: List[int]
    use_sliding_window: bool
    trim_warmup: bool
    future: asyncio.Future
    submitted_at: float

class BiCodecDecoder:
    """
    BiCodec Decoder for Spark TTS.

    Decodes semantic and global tokens to audio waveforms using BiCodec.
    This replaces the SNAC decoder used in the old Indic Orpheus model.

    Supports streaming with sliding window approach (like SNAC).
    """

    def __init__(
        self,
        device: str = "cuda",
        model_path: str = BICODEC_TOKENIZER_PATH,
        enable_batching: bool = False,  # For compatibility with streaming pipeline
    ):
        """
        Initialize BiCodec decoder.

        Args:
            device: Device for BiCodec model (cuda/cpu)
            model_path: Path to BiCodec model checkpoint
            enable_batching: Enable async batching for streaming (not used yet)
        """
        self.device = torch.device(device if torch.cuda.is_available() else "cpu")
        self.model_path = model_path
        requested_batching = bool(enable_batching)
        if _env_flag("VEENA3_BICODEC_BATCHING_FORCE_ON", False):
            requested_batching = True
        if _env_flag("VEENA3_BICODEC_BATCHING_FORCE_OFF", False):
            requested_batching = False
        self.enable_batching = requested_batching and self.device.type == "cuda"
        self.batch_max_size = _env_int("VEENA3_BICODEC_BATCH_MAX", 16, minimum=1)
        self.batch_timeout_s = _env_float("VEENA3_BICODEC_BATCH_TIMEOUT_MS", 6.0, minimum=0.1) / 1000.0

        # Async micro-batcher state (per event loop).
        self._batch_map_lock = threading.Lock()
        self._batch_queues: Dict[int, asyncio.Queue] = {}
        self._batch_workers: Dict[int, asyncio.Task] = {}
        self._batch_total_batches = 0
        self._batch_total_requests = 0
        self._batch_max_seen = 0

        print(f"🎵 Loading BiCodec audio tokenizer from {model_path}...")
        print(f"   Device: {self.device}")

        # Initialize BiCodec tokenizer
        self.audio_tokenizer = BiCodecTokenizer(model_path, device=str(self.device))

        # OPTIMIZATION: Ensure model is on device once at init, not per-decode call
        self.audio_tokenizer.device = self.device
        self.audio_tokenizer.model.to(self.device)

        # NOTE: torch.compile disabled for BiCodec -- dynamic semantic sequence lengths
        # cause repeated recompilation (~120s each), worse than eager mode.
        # torch.compile works best with static shapes. If needed, pre-warm with
        # representative lengths or use mode="default" instead of "reduce-overhead".
        # The multi-engine approach (Tier 3) provides more impactful gains.

        print(f"✅ BiCodec decoder initialized (sample rate: {AUDIO_SAMPLE_RATE}Hz)")
        if self.enable_batching:
            print(
                "   ℹ  Micro-batching enabled "
                f"(max_batch={self.batch_max_size}, timeout_ms={self.batch_timeout_s * 1000:.1f})"
            )

    def decode(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
    ) -> Optional[np.ndarray]:
        """
        Decode BiCodec tokens to audio waveform.

        CRITICAL: BiCodec expects EXACTLY 32 global tokens!
        The decoder pools them via speaker_encoder → d_vector, then broadcasts
        via d_vector.unsqueeze(-1) across the time dimension.

        Args:
            semantic_ids: List of semantic token IDs (variable length)
            global_ids: List of global token IDs (MUST be exactly 32!)

        Returns:
            Audio waveform as numpy array (float32, 16kHz mono)
            Shape: (samples,)
            Returns None if decode fails
        """
        if not semantic_ids or not global_ids:
            print(f"⚠  Empty token lists: semantic={len(semantic_ids)}, global={len(global_ids)}")
            return None

        # Validate global token count
        EXPECTED_GLOBAL_TOKENS = 32
        if len(global_ids) != EXPECTED_GLOBAL_TOKENS:
            print(f"❌ BiCodec requires EXACTLY {EXPECTED_GLOBAL_TOKENS} global tokens, got {len(global_ids)}")
            # Try to fix it
            if len(global_ids) > EXPECTED_GLOBAL_TOKENS:
                global_ids = global_ids[:EXPECTED_GLOBAL_TOKENS]
                print(f"   └─ Truncated to {EXPECTED_GLOBAL_TOKENS}")
            else:
                global_ids = global_ids + [0] * (EXPECTED_GLOBAL_TOKENS - len(global_ids))
                print(f"   └─ Padded to {EXPECTED_GLOBAL_TOKENS}")

        try:
            # Convert to tensors following official Spark-TTS implementation
            # global_token_ids shape: (1, 32) - ALWAYS 32!
            # pred_semantic_ids shape: (1, N) - variable length
            pred_semantic = torch.tensor(semantic_ids).long().unsqueeze(0).to(self.device)
            pred_global = torch.tensor(global_ids).long().unsqueeze(0).to(self.device)

            # Device migration done once in __init__, not per-call
            # Decode through BiCodec
            # BiCodecTokenizer.detokenize() expects:
            #   - global_tokens: (batch, global_dim) -> will unsqueeze(1) internally
            #   - semantic_tokens: (batch, latent_dim)
            with torch.inference_mode():
                wav_np = self.audio_tokenizer.detokenize(
                    pred_global,  # (1, seq) - correct shape!
                    pred_semantic  # (1, seq)
                )

            return wav_np

        except Exception as e:
            print(f"❌ BiCodec decode error: {e}")
            import traceback
            traceback.print_exc()
            return None

    def decode_to_bytes(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
    ) -> Optional[bytes]:
        """
        Decode BiCodec tokens to audio bytes (int16 PCM).

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs

        Returns:
            Audio as bytes (int16 PCM, 16kHz mono)
            Returns None if decode fails
        """
        audio = self.decode(semantic_ids, global_ids)

        if audio is None:
            return None

        # Clip and convert float32 to int16 PCM
        audio = np.clip(audio, -1.0, 1.0)
        audio_int16 = (audio * 32767).astype(np.int16)

        return audio_int16.tobytes()

    async def decode_to_bytes_async(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
    ) -> Optional[bytes]:
        """
        Async wrapper for decode_to_bytes.

        Runs decode in the default executor so the event loop stays responsive
        under high request concurrency.
        """
        loop = asyncio.get_running_loop()
        return await loop.run_in_executor(
            None,
            self.decode_to_bytes,
            semantic_ids,
            global_ids,
        )

    @staticmethod
    def _normalize_global_ids(global_ids: List[int]) -> List[int]:


sed -n '260,560p' veena3modal/core/bicodec_decoder.py


Output:
Chunk ID: cebf71
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 3186
Output:
    def _normalize_global_ids(global_ids: List[int]) -> List[int]:
        expected = 32
        if len(global_ids) >= expected:
            return list(global_ids[:expected])
        return list(global_ids) + [0] * (expected - len(global_ids))

    def _decode_streaming_batch_sync(
        self,
        batch_items: List[Tuple[List[int], List[int], bool, bool]],
    ) -> List[Optional[bytes]]:
        """
        Run a single batched BiCodec forward pass for multiple stream decode requests.

        Falls back to sequential decode_streaming if shapes or settings are not batch-safe.
        """
        if not batch_items:
            return []
        if len(batch_items) == 1:
            sem, glob, use_sw, trim = batch_items[0]
            return [self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)]

        # Sliding window path uses special trimming behavior; keep it sequential.
        if any(use_sw for _, _, use_sw, _ in batch_items):
            return [
                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
                for sem, glob, use_sw, trim in batch_items
            ]

        # Decoder needs at least 8 semantic tokens. Use fallback if any request is below.
        if any(len(sem) < 8 for sem, _, _, _ in batch_items):
            return [
                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
                for sem, glob, use_sw, trim in batch_items
            ]

        try:
            batch_size = len(batch_items)
            max_sem_len = max(len(sem) for sem, _, _, _ in batch_items)

            semantic_batch = torch.zeros(
                (batch_size, max_sem_len),
                dtype=torch.long,
                device=self.device,
            )
            global_batch = torch.zeros(
                (batch_size, 32),
                dtype=torch.long,
                device=self.device,
            )
            semantic_lengths: List[int] = []

            for idx, (semantic_ids, global_ids, _, _) in enumerate(batch_items):
                sem_len = len(semantic_ids)
                semantic_lengths.append(sem_len)
                semantic_batch[idx, :sem_len] = torch.tensor(
                    semantic_ids,
                    dtype=torch.long,
                    device=self.device,
                )
                global_batch[idx] = torch.tensor(
                    self._normalize_global_ids(global_ids),
                    dtype=torch.long,
                    device=self.device,
                )

            with torch.inference_mode():
                wav_batch = self.audio_tokenizer.detokenize(global_batch, semantic_batch)

            results: List[Optional[bytes]] = []
            for idx, sem_len in enumerate(semantic_lengths):
                audio = wav_batch[idx] if getattr(wav_batch, "ndim", 1) > 1 else wav_batch
                if isinstance(audio, torch.Tensor):
                    audio = audio.detach().cpu().numpy()
                audio_np = np.asarray(audio).reshape(-1)
                expected_samples = sem_len * 320
                if expected_samples > 0 and len(audio_np) > expected_samples:
                    audio_np = audio_np[:expected_samples]
                audio_np = np.clip(audio_np, -1.0, 1.0)
                audio_int16 = (audio_np * 32767).astype(np.int16)
                results.append(audio_int16.tobytes())
            return results
        except Exception:
            logger.exception("BiCodec batched decode failed; falling back to sequential decode")
            return [
                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
                for sem, glob, use_sw, trim in batch_items
            ]

    def _ensure_batch_worker(self, loop: asyncio.AbstractEventLoop) -> asyncio.Queue:
        key = id(loop)
        with self._batch_map_lock:
            queue = self._batch_queues.get(key)
            worker = self._batch_workers.get(key)
            if queue is not None and worker is not None and not worker.done():
                return queue

            queue = asyncio.Queue(maxsize=max(self.batch_max_size * 8, 256))
            worker = loop.create_task(self._batch_worker(loop, queue), name=f"bicodec-batch-worker-{key}")
            self._batch_queues[key] = queue
            self._batch_workers[key] = worker
            return queue

    async def _batch_worker(self, loop: asyncio.AbstractEventLoop, queue: asyncio.Queue) -> None:
        key = id(loop)
        try:
            while True:
                first: _BatchDecodeRequest = await queue.get()
                batch: List[_BatchDecodeRequest] = [first]
                deadline = loop.time() + self.batch_timeout_s

                while len(batch) < self.batch_max_size:
                    remaining = deadline - loop.time()
                    if remaining <= 0:
                        break
                    try:
                        req: _BatchDecodeRequest = await asyncio.wait_for(queue.get(), timeout=remaining)
                    except asyncio.TimeoutError:
                        break
                    batch.append(req)

                batch_inputs = [
                    (req.semantic_ids, req.global_ids, req.use_sliding_window, req.trim_warmup)
                    for req in batch
                ]
                t0 = time.perf_counter()
                try:
                    results = await loop.run_in_executor(None, self._decode_streaming_batch_sync, batch_inputs)
                    if len(results) != len(batch):
                        raise RuntimeError("batch decode result size mismatch")
                    for req, result in zip(batch, results):
                        if not req.future.done():
                            req.future.set_result(result)
                except Exception as exc:
                    for req in batch:
                        if not req.future.done():
                            req.future.set_exception(exc)
                finally:
                    _ = (time.perf_counter() - t0) * 1000.0
                    self._batch_total_batches += 1
                    self._batch_total_requests += len(batch)
                    if len(batch) > self._batch_max_seen:
                        self._batch_max_seen = len(batch)
        finally:
            with self._batch_map_lock:
                self._batch_queues.pop(key, None)
                self._batch_workers.pop(key, None)

    def get_batching_stats(self) -> Dict[str, float]:
        batches = max(1, self._batch_total_batches)
        return {
            "enabled": 1.0 if self.enable_batching else 0.0,
            "batch_total_batches": float(self._batch_total_batches),
            "batch_total_requests": float(self._batch_total_requests),
            "batch_avg_size": float(self._batch_total_requests / batches),
            "batch_max_seen": float(self._batch_max_seen),
        }

    def validate_tokens(self, semantic_ids: List[int], global_ids: List[int]) -> bool:
        """
        Validate BiCodec tokens before decoding.

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs

        Returns:
            True if valid, False otherwise
        """
        # Check minimum length
        if not semantic_ids:
            print(f"❌ No semantic tokens")
            return False

        if not global_ids:
            print(f"❌ No global tokens")
            return False

        # Basic sanity checks
        if len(semantic_ids) < 1:
            print(f"❌ Too few semantic tokens: {len(semantic_ids)}")
            return False

        if len(global_ids) < 1:
            print(f"❌ Too few global tokens: {len(global_ids)}")
            return False

        return True

    def get_audio_duration(self, semantic_ids: List[int]) -> float:
        """
        Estimate audio duration from semantic tokens.

        Args:
            semantic_ids: List of semantic token IDs

        Returns:
            Estimated duration in seconds
        """
        # BiCodec generates audio based on semantic tokens
        # Approximate duration calculation (may need tuning)
        # This is an estimate and actual duration depends on model output
        estimated_samples = len(semantic_ids) * 320  # Rough estimate
        return estimated_samples / AUDIO_SAMPLE_RATE

    def decode_streaming(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
        use_sliding_window: bool = False,
        trim_warmup: bool = False,
    ) -> Optional[bytes]:
        """
        Decode BiCodec tokens with streaming support.

        CRITICAL: BiCodec decoder expects EXACTLY 32 global tokens always!
        The decoder internally broadcasts d_vector via d_vector.unsqueeze(-1).

        From sparktts/models/bicodec.py line 184-186:
            d_vector = self.speaker_encoder.detokenize(global_tokens)  # Expects 32 tokens
            x = self.prenet(z_q, d_vector)
            x = x + d_vector.unsqueeze(-1)  # Broadcasts across time automatically!

        Args:
            semantic_ids: List of semantic token IDs (variable length, 50 TPS)
            global_ids: List of global token IDs (MUST be exactly 32 tokens!)
            use_sliding_window: If True, return only middle samples
            trim_warmup: Legacy parameter (not used for BiCodec)

        Returns:
            Audio bytes (int16 PCM) or None if decode fails
        """
        # Minimum semantic tokens needed for stable decode
        MIN_SEMANTIC_TOKENS = 8

        if len(semantic_ids) < MIN_SEMANTIC_TOKENS:
            print(f"⚠  Too few semantic tokens ({len(semantic_ids)}) for decode, need >= {MIN_SEMANTIC_TOKENS}")
            return None

        # CRITICAL: Ensure we have exactly 32 global tokens
        EXPECTED_GLOBAL_TOKENS = 32
        if len(global_ids) != EXPECTED_GLOBAL_TOKENS:
            print(f"⚠  WARNING: Got {len(global_ids)} global tokens, expected {EXPECTED_GLOBAL_TOKENS}")
            if len(global_ids) > EXPECTED_GLOBAL_TOKENS:
                # Truncate to first 32
                global_ids = global_ids[:EXPECTED_GLOBAL_TOKENS]
                print(f"   └─ Truncated to {EXPECTED_GLOBAL_TOKENS} tokens")
            else:
                # Pad with zeros if we somehow have fewer
                global_ids = global_ids + [0] * (EXPECTED_GLOBAL_TOKENS - len(global_ids))
                print(f"   └─ Padded to {EXPECTED_GLOBAL_TOKENS} tokens")

        # Decode with EXACTLY 32 global tokens + variable semantic tokens
        # The decoder handles broadcasting internally!
        audio = self.decode(semantic_ids, global_ids)

        if audio is None:
            return None

        # If using sliding window, return only the middle portion
        if use_sliding_window and len(audio) > 4096:
            # Keep middle samples for overlap-add streaming
            # BiCodec produces ~320 samples per semantic token (16kHz / 50 TPS)
            total_samples = len(audio)
            keep_samples = min(4096, total_samples // 2)
            start = (total_samples - keep_samples) // 2
            end = start + keep_samples
            audio = audio[start:end]

        # Convert to bytes
        audio = np.clip(audio, -1.0, 1.0)
        audio_int16 = (audio * 32767).astype(np.int16)

        return audio_int16.tobytes()

    async def decode_single_async(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
        trim_warmup: bool = False,
        use_sliding_window: bool = False,
    ) -> Optional[bytes]:
        """
        Async wrapper for streaming decode - runs in executor to avoid blocking event loop.

        OPTIMIZATION: Previously ran synchronously, blocking all concurrent coroutines
        during GPU decode (~20-50ms). Now uses run_in_executor to unblock the event loop
        so other streams can progress while this decode runs on GPU.

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs
            trim_warmup: Legacy parameter (not used)
            use_sliding_window: Use sliding window mode

        Returns:
            Audio bytes (int16 PCM) or None
        """
        loop = asyncio.get_running_loop()
        if self.enable_batching and not use_sliding_window:
            queue = self._ensure_batch_worker(loop)
            future: asyncio.Future = loop.create_future()


sed -n '560,760p' veena3modal/core/bicodec_decoder.py


Output:
Chunk ID: 1cbe1b
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 166
Output:
            future: asyncio.Future = loop.create_future()
            req = _BatchDecodeRequest(
                semantic_ids=list(semantic_ids),
                global_ids=list(global_ids),
                use_sliding_window=use_sliding_window,
                trim_warmup=trim_warmup,
                future=future,
                submitted_at=time.perf_counter(),
            )
            await queue.put(req)
            return await future

        return await loop.run_in_executor(
            None,
            self.decode_streaming,
            semantic_ids,
            global_ids,
            use_sliding_window,
            trim_warmup,
        )


**Designing adaptive decode stats and logic**


Output:
Chunk ID: 1d3500
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 649
Output:
66:_STREAM_MIN_SEMANTIC_FIRST_CHUNK = _env_int("VEENA3_STREAM_MIN_SEMANTIC_FIRST", 10, minimum=1)
67:_STREAM_DECODE_INTERVAL = _env_int("VEENA3_STREAM_DECODE_INTERVAL", 48, minimum=1)
68:_STREAM_CROSSFADE_MS = _env_int("VEENA3_STREAM_CROSSFADE_MS", 50, minimum=0)
69:_STREAM_WINDOW_SIZE = _env_int("VEENA3_STREAM_WINDOW_SIZE", 128, minimum=16)
482:                if semantic_count >= _STREAM_MIN_SEMANTIC_FIRST_CHUNK:
486:                    if semantic_count - last_decode_count >= _STREAM_DECODE_INTERVAL:
492:                            and semantic_count > _STREAM_WINDOW_SIZE
494:                            decode_semantic = semantic_buffer[-_STREAM_WINDOW_SIZE:]
518:                                and semantic_count > _STREAM_WINDOW_SIZE
520:                                new_samples_approx = _STREAM_DECODE_INTERVAL * 320
538:                                    crossfade_ms=_STREAM_CROSSFADE_MS,
583:                        crossfade_ms=_STREAM_CROSSFADE_MS,
628:            "stream_decode_interval_tokens": int(_STREAM_DECODE_INTERVAL),
629:            "stream_window_size_tokens": int(_STREAM_WINDOW_SIZE),
808:        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
809:        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
810:        CROSSFADE_MS = 50
850:                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
851:                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
883:                                    crossfade_ms=CROSSFADE_MS,
913:                        crossfade_ms=CROSSFADE_MS,
1022:        MIN_SEMANTIC_FOR_FIRST_CHUNK = _STREAM_MIN_SEMANTIC_FIRST_CHUNK
1023:        DECODE_INTERVAL = _STREAM_DECODE_INTERVAL
1024:        CROSSFADE_MS = _STREAM_CROSSFADE_MS
1025:        WINDOW_SIZE = _STREAM_WINDOW_SIZE
1063:            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
1064:                if semantic_count - last_decode_count >= DECODE_INTERVAL:
1066:                    if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
1067:                        decode_semantic = semantic_buffer[-WINDOW_SIZE:]
1092:                        if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
1093:                            new_samples_approx = DECODE_INTERVAL * 320
1110:                                crossfade_ms=CROSSFADE_MS,
1145:                        crossfade_ms=CROSSFADE_MS,
1192:                "stream_decode_interval_tokens": int(DECODE_INTERVAL),
1193:                "stream_window_size_tokens": int(WINDOW_SIZE),


**Inspecting duplicate method for patching**


sed -n '980,1235p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: ea1d62
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2690
Output:
            ValueError: If global_ids doesn't contain exactly 32 tokens

        Thread Safety:
            Fully thread-safe. All state is request-scoped and passed explicitly.
        """
        import time
        t_start = time.time()

        # Validate global tokens
        if len(global_ids) != _STREAM_EXPECTED_GLOBAL_COUNT:
            raise ValueError(
                f"Expected exactly {_STREAM_EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
                f"global_ids must be captured from first chunk generation."
            )

        # Build prompt WITH pre-filled global tokens
        # This tells the model to skip global generation and go straight to semantic tokens
        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)

        # Configure sampling
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,
            seed=seed,
            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
        )

        # BiCodec token buffers
        semantic_buffer = []
        global_buffer_unused = []  # Not used in continuation, but needed for parser API
        processed_token_count = 0  # Used for cumulative mode; ignored for delta mode.
        total_audio_chunks = 0
        total_generated_tokens = 0

        # BiCodec streaming configuration
        MIN_SEMANTIC_FOR_FIRST_CHUNK = _STREAM_MIN_SEMANTIC_FIRST_CHUNK
        DECODE_INTERVAL = _STREAM_DECODE_INTERVAL
        CROSSFADE_MS = _STREAM_CROSSFADE_MS
        WINDOW_SIZE = _STREAM_WINDOW_SIZE

        # Generate unique request ID
        import uuid
        request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None
        t_first_chunk_emitted: Optional[float] = None
        decode_calls = 0
        decode_wall_ms_total = 0.0

        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
        async for request_output in results_generator:
            generated_ids = request_output.outputs[0].token_ids

            # In DELTA mode token_ids are already incremental.
            if _STREAM_USE_DELTA_OUTPUT:
                new_token_ids = generated_ids
                processed_token_count += len(new_token_ids)
            else:
                new_token_ids = generated_ids[processed_token_count:]
                processed_token_count = len(generated_ids)
            total_generated_tokens += len(new_token_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)

            semantic_count = len(semantic_buffer)
            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                if semantic_count - last_decode_count >= DECODE_INTERVAL:
                    last_decode_count = semantic_count
                    if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
                        decode_semantic = semantic_buffer[-WINDOW_SIZE:]
                    else:
                        decode_semantic = semantic_buffer

                    # Use pre-captured global tokens for decoding
                    decode_t0 = time.perf_counter()
                    if self.snac_decoder.enable_batching:
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=decode_semantic,
                            global_ids=global_ids,  # Use captured globals
                            trim_warmup=False,
                            use_sliding_window=False
                        )
                    else:
                        audio_bytes = self.snac_decoder.decode_streaming(
                            semantic_ids=decode_semantic,
                            global_ids=global_ids,  # Use captured globals
                            use_sliding_window=False,
                            trim_warmup=False
                        )
                    decode_calls += 1
                    decode_wall_ms_total += (time.perf_counter() - decode_t0) * 1000.0

                    if audio_bytes:
                        total_samples_decoded = len(audio_bytes) // 2
                        if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
                            new_samples_approx = DECODE_INTERVAL * 320
                            if total_samples_decoded > new_samples_approx:
                                new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
                            else:
                                new_bytes_start = 0
                            new_audio_bytes = audio_bytes[new_bytes_start:]
                        elif total_samples_decoded > total_samples_emitted_to_user:
                            new_bytes_start = total_samples_emitted_to_user * 2
                            new_audio_bytes = audio_bytes[new_bytes_start:]
                        else:
                            new_audio_bytes = b""

                        if new_audio_bytes:
                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                previous_chunk_tail,
                                new_audio_bytes,
                                sample_rate_hz=16000,
                                crossfade_ms=CROSSFADE_MS,
                            )

                            if to_emit:
                                total_audio_chunks += 1
                                total_samples_emitted_to_user += len(to_emit) // 2
                                if t_first_chunk_emitted is None:
                                    t_first_chunk_emitted = time.perf_counter()
                                if emit_progress:
                                    yield (to_emit, {})
                                else:
                                    yield to_emit

        # Final chunk processing
        if len(semantic_buffer) > last_decode_count:
            decode_t0 = time.perf_counter()
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,
                global_ids=global_ids,  # Use captured globals
                use_sliding_window=False,
                trim_warmup=False
            )
            decode_calls += 1
            decode_wall_ms_total += (time.perf_counter() - decode_t0) * 1000.0
            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    if to_emit:
                        total_audio_chunks += 1
                        total_samples_emitted_to_user += len(to_emit) // 2
                        if t_first_chunk_emitted is None:
                            t_first_chunk_emitted = time.perf_counter()
                        if emit_progress:
                            yield (to_emit, {})
                        else:
                            yield to_emit

        # Flush remaining tail
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            if t_first_chunk_emitted is None:
                t_first_chunk_emitted = time.perf_counter()
            if emit_progress:
                yield (previous_chunk_tail, {})
            else:
                yield previous_chunk_tail

        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000
        logger.debug(
            "continuation complete audio=%.2fs chunks=%d",
            audio_duration_s,
            total_audio_chunks,
        )

        if emit_progress:
            perf: Dict[str, Any] = {
                "llm_token_total": int(total_generated_tokens),
                "semantic_token_total": int(len(semantic_buffer)),
                "global_token_total": int(len(global_ids)),
                "bicodec_decode_calls": int(decode_calls),
                "llm_decode_calls": int(decode_calls),
                "bicodec_decode_wall_ms": float(decode_wall_ms_total),
                "bicodec_decode_gpu_ms": 0.0,
                "bicodec_decode_cpu_ms": float(decode_wall_ms_total),
                "chunks_sent": int(total_audio_chunks),
                "audio_duration_seconds": float(audio_duration_s),
                "ttfb_ms": float((t_first_chunk_emitted - t_start) * 1000.0 if t_first_chunk_emitted else 0.0),
                "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
                "stream_decode_interval_tokens": int(DECODE_INTERVAL),
                "stream_window_size_tokens": int(WINDOW_SIZE),
                "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
            }
            decoder_batch_stats = {}
            if hasattr(self.snac_decoder, "get_batching_stats"):
                try:
                    decoder_batch_stats = self.snac_decoder.get_batching_stats() or {}
                except Exception:
                    decoder_batch_stats = {}
            if decoder_batch_stats:
                perf.update(decoder_batch_stats)
            yield (b"", perf)


**Adding adaptive decode interval logic**