Output:
Chunk ID: a3e75f
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 1608
Output:
   100	        """
   101	        Initialize BiCodec decoder.
   102
   103	        Args:
   104	            device: Device for BiCodec model (cuda/cpu)
   105	            model_path: Path to BiCodec model checkpoint
   106	            enable_batching: Enable async batching for streaming (not used yet)
   107	        """
   108	        self.device = torch.device(device if torch.cuda.is_available() else "cpu")
   109	        self.model_path = model_path
   110	        requested_batching = bool(enable_batching)
   111	        if _env_flag("VEENA3_BICODEC_BATCHING_FORCE_ON", False):
   112	            requested_batching = True
   113	        if _env_flag("VEENA3_BICODEC_BATCHING_FORCE_OFF", False):
   114	            requested_batching = False
   115	        self.enable_batching = requested_batching and self.device.type == "cuda"
   116	        self.batch_max_size = _env_int("VEENA3_BICODEC_BATCH_MAX", 16, minimum=1)
   117	        self.batch_timeout_s = _env_float("VEENA3_BICODEC_BATCH_TIMEOUT_MS", 6.0, minimum=0.1) / 1000.0
   118
   119	        # Async micro-batcher state (per event loop).
   120	        self._batch_map_lock = threading.Lock()
   121	        self._batch_queues: Dict[int, asyncio.Queue] = {}
   122	        self._batch_workers: Dict[int, asyncio.Task] = {}
   123	        self._batch_total_batches = 0
   124	        self._batch_total_requests = 0
   125	        self._batch_max_seen = 0
   126	        self._batch_queue_wait_ms_total = 0.0
   127	        self._batch_queue_wait_ms_min: Optional[float] = None
   128	        self._batch_queue_wait_ms_max = 0.0
   129	        self._batch_compute_ms_total = 0.0
   130	        self._batch_compute_ms_min: Optional[float] = None
   131	        self._batch_compute_ms_max = 0.0
   132	        self._batch_queue_depth_total = 0
   133	        self._batch_queue_depth_max = 0
   134	        self._batch_inflight_requests = 0
   135
   136	        print(f"🎵 Loading BiCodec audio tokenizer from {model_path}...")
   137	        print(f"   Device: {self.device}")
   138
   139	        # Initialize BiCodec tokenizer
   140	        self.audio_tokenizer = BiCodecTokenizer(model_path, device=str(self.device))
   141
   142	        # OPTIMIZATION: Ensure model is on device once at init, not per-decode call
   143	        self.audio_tokenizer.device = self.device
   144	        self.audio_tokenizer.model.to(self.device)
   145
   146	        # NOTE: torch.compile disabled for BiCodec -- dynamic semantic sequence lengths
   147	        # cause repeated recompilation (~120s each), worse than eager mode.
   148	        # torch.compile works best with static shapes. If needed, pre-warm with
   149	        # representative lengths or use mode="default" instead of "reduce-overhead".
   150	        # The multi-engine approach (Tier 3) provides more impactful gains.
   151
   152	        print(f"✅ BiCodec decoder initialized (sample rate: {AUDIO_SAMPLE_RATE}Hz)")
   153	        if self.enable_batching:
   154	            print(
   155	                "   ℹ  Micro-batching enabled "
   156	                f"(max_batch={self.batch_max_size}, timeout_ms={self.batch_timeout_s * 1000:.1f})"
   157	            )
   158
   159	    def decode(
   160	        self,
   161	        semantic_ids: List[int],
   162	        global_ids: List[int],
   163	    ) -> Optional[np.ndarray]:
   164	        """
   165	        Decode BiCodec tokens to audio waveform.
   166
   167	        CRITICAL: BiCodec expects EXACTLY 32 global tokens!
   168	        The decoder pools them via speaker_encoder → d_vector, then broadcasts
   169	        via d_vector.unsqueeze(-1) across the time dimension.
   170
   171	        Args:
   172	            semantic_ids: List of semantic token IDs (variable length)
   173	            global_ids: List of global token IDs (MUST be exactly 32!)
   174
   175	        Returns:
   176	            Audio waveform as numpy array (float32, 16kHz mono)
   177	            Shape: (samples,)
   178	            Returns None if decode fails
   179	        """
   180	        if not semantic_ids or not global_ids:
   181	            print(f"⚠  Empty token lists: semantic={len(semantic_ids)}, global={len(global_ids)}")
   182	            return None
   183
   184	        # Validate global token count
   185	        EXPECTED_GLOBAL_TOKENS = 32
   186	        if len(global_ids) != EXPECTED_GLOBAL_TOKENS:
   187	            print(f"❌ BiCodec requires EXACTLY {EXPECTED_GLOBAL_TOKENS} global tokens, got {len(global_ids)}")
   188	            # Try to fix it
   189	            if len(global_ids) > EXPECTED_GLOBAL_TOKENS:
   190	                global_ids = global_ids[:EXPECTED_GLOBAL_TOKENS]
   191	                print(f"   └─ Truncated to {EXPECTED_GLOBAL_TOKENS}")
   192	            else:
   193	                global_ids = global_ids + [0] * (EXPECTED_GLOBAL_TOKENS - len(global_ids))
   194	                print(f"   └─ Padded to {EXPECTED_GLOBAL_TOKENS}")
   195
   196	        try:
   197	            # Convert to tensors following official Spark-TTS implementation
   198	            # global_token_ids shape: (1, 32) - ALWAYS 32!
   199	            # pred_semantic_ids shape: (1, N) - variable length
   200	            pred_semantic = torch.tensor(semantic_ids).long().unsqueeze(0).to(self.device)
   201	            pred_global = torch.tensor(global_ids).long().unsqueeze(0).to(self.device)
   202
   203	            # Device migration done once in __init__, not per-call
   204	            # Decode through BiCodec
   205	            # BiCodecTokenizer.detokenize() expects:
   206	            #   - global_tokens: (batch, global_dim) -> will unsqueeze(1) internally
   207	            #   - semantic_tokens: (batch, latent_dim)
   208	            with torch.inference_mode():
   209	                wav_np = self.audio_tokenizer.detokenize(
   210	                    pred_global,  # (1, seq) - correct shape!
   211	                    pred_semantic  # (1, seq)
   212	                )
   213
   214	            return wav_np
   215
   216	        except Exception as e:
   217	            print(f"❌ BiCodec decode error: {e}")
   218	            import traceback
   219	            traceback.print_exc()
   220	            return None


Output:
Chunk ID: 9476d0
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1920
Output:
   340	                if isinstance(audio, torch.Tensor):
   341	                    audio = audio.detach().cpu().numpy()
   342	                audio_np = np.asarray(audio).reshape(-1)
   343	                expected_samples = sem_len * 320
   344	                if expected_samples > 0 and len(audio_np) > expected_samples:
   345	                    audio_np = audio_np[:expected_samples]
   346	                audio_np = np.clip(audio_np, -1.0, 1.0)
   347	                audio_int16 = (audio_np * 32767).astype(np.int16)
   348	                results.append(audio_int16.tobytes())
   349	            return results
   350	        except Exception:
   351	            logger.exception("BiCodec batched decode failed; falling back to sequential decode")
   352	            return [
   353	                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
   354	                for sem, glob, use_sw, trim in batch_items
   355	            ]
   356
   357	    def _ensure_batch_worker(self, loop: asyncio.AbstractEventLoop) -> asyncio.Queue:
   358	        key = id(loop)
   359	        with self._batch_map_lock:
   360	            queue = self._batch_queues.get(key)
   361	            worker = self._batch_workers.get(key)
   362	            if queue is not None and worker is not None and not worker.done():
   363	                return queue
   364
   365	            queue = asyncio.Queue(maxsize=max(self.batch_max_size * 8, 256))
   366	            worker = loop.create_task(self._batch_worker(loop, queue), name=f"bicodec-batch-worker-{key}")
   367	            self._batch_queues[key] = queue
   368	            self._batch_workers[key] = worker
   369	            return queue
   370
   371	    async def _batch_worker(self, loop: asyncio.AbstractEventLoop, queue: asyncio.Queue) -> None:
   372	        key = id(loop)
   373	        try:
   374	            while True:
   375	                first: _BatchDecodeRequest = await queue.get()
   376	                batch: List[_BatchDecodeRequest] = [first]
   377	                deadline = loop.time() + self.batch_timeout_s
   378
   379	                while len(batch) < self.batch_max_size:
   380	                    remaining = deadline - loop.time()
   381	                    if remaining <= 0:
   382	                        break
   383	                    try:
   384	                        req: _BatchDecodeRequest = await asyncio.wait_for(queue.get(), timeout=remaining)
   385	                    except asyncio.TimeoutError:
   386	                        break
   387	                    batch.append(req)
   388
   389	                batch_inputs = [
   390	                    (req.semantic_ids, req.global_ids, req.use_sliding_window, req.trim_warmup)
   391	                    for req in batch
   392	                ]
   393	                batch_start = time.perf_counter()
   394	                queue_wait_ms = [
   395	                    max(0.0, (batch_start - req.submitted_at) * 1000.0)
   396	                    for req in batch
   397	                ]
   398	                queue_depth_now = queue.qsize()
   399	                self._batch_queue_depth_total += queue_depth_now
   400	                if queue_depth_now > self._batch_queue_depth_max:
   401	                    self._batch_queue_depth_max = queue_depth_now
   402	                for wait_ms in queue_wait_ms:
   403	                    self._batch_queue_wait_ms_total += wait_ms
   404	                    if self._batch_queue_wait_ms_min is None or wait_ms < self._batch_queue_wait_ms_min:
   405	                        self._batch_queue_wait_ms_min = wait_ms
   406	                    if wait_ms > self._batch_queue_wait_ms_max:
   407	                        self._batch_queue_wait_ms_max = wait_ms
   408	                with self._batch_map_lock:
   409	                    self._batch_inflight_requests += len(batch)
   410
   411	                t0 = time.perf_counter()
   412	                try:
   413	                    results = await loop.run_in_executor(None, self._decode_streaming_batch_sync, batch_inputs)
   414	                    if len(results) != len(batch):
   415	                        raise RuntimeError("batch decode result size mismatch")
   416	                    for req, result in zip(batch, results):
   417	                        if not req.future.done():
   418	                            req.future.set_result(result)
   419	                except Exception as exc:
   420	                    for req in batch:
   421	                        if not req.future.done():
   422	                            req.future.set_exception(exc)
   423	                finally:
   424	                    compute_ms = (time.perf_counter() - t0) * 1000.0
   425	                    self._batch_compute_ms_total += compute_ms
   426	                    if self._batch_compute_ms_min is None or compute_ms < self._batch_compute_ms_min:
   427	                        self._batch_compute_ms_min = compute_ms
   428	                    if compute_ms > self._batch_compute_ms_max:
   429	                        self._batch_compute_ms_max = compute_ms
   430	                    self._batch_total_batches += 1
   431	                    self._batch_total_requests += len(batch)
   432	                    if len(batch) > self._batch_max_seen:
   433	                        self._batch_max_seen = len(batch)
   434	                    with self._batch_map_lock:
   435	                        self._batch_inflight_requests = max(0, self._batch_inflight_requests - len(batch))
   436	        finally:
   437	            with self._batch_map_lock:
   438	                self._batch_queues.pop(key, None)
   439	                self._batch_workers.pop(key, None)
   440
   441	    def get_pending_requests(self) -> int:
   442	        if not self.enable_batching:
   443	            return 0
   444	        with self._batch_map_lock:
   445	            pending = sum(queue.qsize() for queue in self._batch_queues.values())
   446	            return int(max(0, pending + self._batch_inflight_requests))
   447
   448	    def get_batching_stats(self) -> Dict[str, float]:
   449	        batches = max(1, self._batch_total_batches)
   450	        reqs = max(1, self._batch_total_requests)
   451	        return {
   452	            "enabled": 1.0 if self.enable_batching else 0.0,
   453	            "batch_total_batches": float(self._batch_total_batches),
   454	            "batch_total_requests": float(self._batch_total_requests),
   455	            "batch_avg_size": float(self._batch_total_requests / batches),
   456	            "batch_max_seen": float(self._batch_max_seen),
   457	            "batch_queue_wait_ms_avg": float(self._batch_queue_wait_ms_total / reqs),
   458	            "batch_queue_wait_ms_min": float(self._batch_queue_wait_ms_min or 0.0),
   459	            "batch_queue_wait_ms_max": float(self._batch_queue_wait_ms_max),
   460	            "batch_compute_ms_avg": float(self._batch_compute_ms_total / batches),
   461	            "batch_compute_ms_min": float(self._batch_compute_ms_min or 0.0),
   462	            "batch_compute_ms_max": float(self._batch_compute_ms_max),
   463	            "batch_queue_depth_avg": float(self._batch_queue_depth_total / batches),
   464	            "batch_queue_depth_max": float(self._batch_queue_depth_max),
   465	            "batch_pending_now": float(self.get_pending_requests()),
   466	        }
   467
   468	    def validate_tokens(self, semantic_ids: List[int], global_ids: List[int]) -> bool:
   469	        """
   470	        Validate BiCodec tokens before decoding.


Output:
Chunk ID: 889b24
Wall time: 0.0516 seconds
Process exited with code 0
Original token count: 1063
Output:
   470	        Validate BiCodec tokens before decoding.
   471
   472	        Args:
   473	            semantic_ids: List of semantic token IDs
   474	            global_ids: List of global token IDs
   475
   476	        Returns:
   477	            True if valid, False otherwise
   478	        """
   479	        # Check minimum length
   480	        if not semantic_ids:
   481	            print(f"❌ No semantic tokens")
   482	            return False
   483
   484	        if not global_ids:
   485	            print(f"❌ No global tokens")
   486	            return False
   487
   488	        # Basic sanity checks
   489	        if len(semantic_ids) < 1:
   490	            print(f"❌ Too few semantic tokens: {len(semantic_ids)}")
   491	            return False
   492
   493	        if len(global_ids) < 1:
   494	            print(f"❌ Too few global tokens: {len(global_ids)}")
   495	            return False
   496
   497	        return True
   498
   499	    def get_audio_duration(self, semantic_ids: List[int]) -> float:
   500	        """
   501	        Estimate audio duration from semantic tokens.
   502
   503	        Args:
   504	            semantic_ids: List of semantic token IDs
   505
   506	        Returns:
   507	            Estimated duration in seconds
   508	        """
   509	        # BiCodec generates audio based on semantic tokens
   510	        # Approximate duration calculation (may need tuning)
   511	        # This is an estimate and actual duration depends on model output
   512	        estimated_samples = len(semantic_ids) * 320  # Rough estimate
   513	        return estimated_samples / AUDIO_SAMPLE_RATE
   514
   515	    def decode_streaming(
   516	        self,
   517	        semantic_ids: List[int],
   518	        global_ids: List[int],
   519	        use_sliding_window: bool = False,
   520	        trim_warmup: bool = False,
   521	    ) -> Optional[bytes]:
   522	        """
   523	        Decode BiCodec tokens with streaming support.
   524
   525	        CRITICAL: BiCodec decoder expects EXACTLY 32 global tokens always!
   526	        The decoder internally broadcasts d_vector via d_vector.unsqueeze(-1).
   527
   528	        From sparktts/models/bicodec.py line 184-186:
   529	            d_vector = self.speaker_encoder.detokenize(global_tokens)  # Expects 32 tokens
   530	            x = self.prenet(z_q, d_vector)
   531	            x = x + d_vector.unsqueeze(-1)  # Broadcasts across time automatically!
   532
   533	        Args:
   534	            semantic_ids: List of semantic token IDs (variable length, 50 TPS)
   535	            global_ids: List of global token IDs (MUST be exactly 32 tokens!)
   536	            use_sliding_window: If True, return only middle samples
   537	            trim_warmup: Legacy parameter (not used for BiCodec)
   538
   539	        Returns:
   540	            Audio bytes (int16 PCM) or None if decode fails
   541	        """
   542	        # Minimum semantic tokens needed for stable decode
   543	        MIN_SEMANTIC_TOKENS = 8
   544
   545	        if len(semantic_ids) < MIN_SEMANTIC_TOKENS:
   546	            print(f"⚠  Too few semantic tokens ({len(semantic_ids)}) for decode, need >= {MIN_SEMANTIC_TOKENS}")
   547	            return None
   548
   549	        # CRITICAL: Ensure we have exactly 32 global tokens
   550	        EXPECTED_GLOBAL_TOKENS = 32
   551	        if len(global_ids) != EXPECTED_GLOBAL_TOKENS:
   552	            print(f"⚠  WARNING: Got {len(global_ids)} global tokens, expected {EXPECTED_GLOBAL_TOKENS}")
   553	            if len(global_ids) > EXPECTED_GLOBAL_TOKENS:
   554	                # Truncate to first 32
   555	                global_ids = global_ids[:EXPECTED_GLOBAL_TOKENS]
   556	                print(f"   └─ Truncated to {EXPECTED_GLOBAL_TOKENS} tokens")
   557	            else:
   558	                # Pad with zeros if we somehow have fewer
   559	                global_ids = global_ids + [0] * (EXPECTED_GLOBAL_TOKENS - len(global_ids))
   560	                print(f"   └─ Padded to {EXPECTED_GLOBAL_TOKENS} tokens")


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Scoping line ranges for new helpers**


nl -ba veena3modal/core/streaming_pipeline.py | sed -n '44,120p'


nl -ba veena3modal/core/streaming_pipeline.py | sed -n '150,235p'


nl -ba veena3modal/core/streaming_pipeline.py | sed -n '500,620p'


nl -ba veena3modal/core/streaming_pipeline.py | sed -n '660,720p'


nl -ba veena3modal/core/streaming_pipeline.py | sed -n '1050,1160p'


nl -ba veena3modal/core/streaming_pipeline.py | sed -n '1240,1298p'


Output:
Chunk ID: e4df9e
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 766
Output:
    44	    DEFAULT_MIN_TOKENS,
    45	    DEFAULT_REPETITION_PENALTY,
    46	    DEFAULT_SEED,
    47	)
    48
    49	logger = logging.getLogger(__name__)
    50	_STREAM_OUTPUT_KIND_RAW = os.environ.get("VEENA3_STREAM_OUTPUT_KIND", "delta").strip().lower()
    51	_STREAM_USE_DELTA_OUTPUT = _STREAM_OUTPUT_KIND_RAW in {"delta", "d", "1", "true", "yes", "on"}
    52
    53
    54	def _env_int(name: str, default: int, minimum: int = 0) -> int:
    55	    raw = os.environ.get(name, "")
    56	    if not raw:
    57	        return default
    58	    try:
    59	        value = int(raw)
    60	    except ValueError:
    61	        return default
    62	    return max(minimum, value)
    63
    64
    65	def _env_flag(name: str, default: bool) -> bool:
    66	    raw = os.environ.get(name, "")
    67	    if not raw:
    68	        return default
    69	    return raw.strip().lower() in {"1", "true", "yes", "on"}
    70
    71
    72	_STREAM_EXPECTED_GLOBAL_COUNT = 32
    73	_STREAM_MIN_SEMANTIC_FIRST_CHUNK = _env_int("VEENA3_STREAM_MIN_SEMANTIC_FIRST", 10, minimum=1)
    74	_STREAM_DECODE_INTERVAL = _env_int("VEENA3_STREAM_DECODE_INTERVAL", 48, minimum=1)
    75	_STREAM_CROSSFADE_MS = _env_int("VEENA3_STREAM_CROSSFADE_MS", 50, minimum=0)
    76	_STREAM_WINDOW_SIZE = _env_int("VEENA3_STREAM_WINDOW_SIZE", 128, minimum=16)
    77	_STREAM_ADAPTIVE_DECODE = _env_flag("VEENA3_STREAM_ADAPTIVE_DECODE", False)
    78	_STREAM_DECODE_INTERVAL_FIRST = _env_int("VEENA3_STREAM_DECODE_INTERVAL_FIRST", _STREAM_DECODE_INTERVAL, minimum=1)
    79	_STREAM_DECODE_INTERVAL_BUSY = _env_int(
    80	    "VEENA3_STREAM_DECODE_INTERVAL_BUSY",
    81	    max(_STREAM_DECODE_INTERVAL, 64),
    82	    minimum=1,
    83	)
    84	_STREAM_DECODE_BUSY_PENDING = _env_int("VEENA3_STREAM_DECODE_BUSY_PENDING", 32, minimum=1)
    85	_STREAM_WINDOWED_DECODE = os.environ.get("VEENA3_STREAM_WINDOWED_DECODE", "true").strip().lower() in {
    86	    "1",
    87	    "true",
    88	    "yes",
    89	    "on",
    90	}
    91
    92
    93	class Veena3SlidingWindowPipeline:
    94	    """
    95	    Streaming TTS pipeline using sliding window approach.
    96
    97	    This eliminates choppy audio and popping artifacts by:
    98	    - Decoding overlapping 28-token windows (4 frames)
    99	    - Keeping only the middle 2048 samples from each decode
   100	    - Creating natural continuity between chunks
   101
   102	    Based on the official Canopy Labs implementation.
   103	    """
   104
   105	    def __init__(
   106	        self,
   107	        model,
   108	        prompt_builder,
   109	        snac_decoder,
   110	    ):
   111	        """
   112	        Initialize sliding window streaming pipeline.
   113
   114	        Args:
   115	            model: Veena3Model instance
   116	            prompt_builder: Veena3PromptBuilder instance
   117	            snac_decoder: SNACDecoder instance (with batching enabled)
   118	        """
   119	        self.model = model
   120	        self.prompt_builder = prompt_builder


Output:
Chunk ID: 07c458
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 912
Output:
   150	    def _extract_request_metrics(self, raw_metrics: Any) -> Dict[str, float]:
   151	        if raw_metrics is None:
   152	            return {}
   153
   154	        numeric: Dict[str, float] = {}
   155	        items: List[Tuple[str, Any]] = []
   156
   157	        if isinstance(raw_metrics, dict):
   158	            items.extend(raw_metrics.items())
   159	        else:
   160	            if hasattr(raw_metrics, "__dict__"):
   161	                try:
   162	                    items.extend(dict(raw_metrics.__dict__).items())
   163	                except Exception:
   164	                    pass
   165	            for name in dir(raw_metrics):
   166	                if name.startswith("_"):
   167	                    continue
   168	                try:
   169	                    value = getattr(raw_metrics, name)
   170	                except Exception:
   171	                    continue
   172	                if callable(value):
   173	                    continue
   174	                items.append((name, value))
   175
   176	        seen: set[str] = set()
   177	        for key, value in items:
   178	            key_s = str(key)
   179	            if key_s in seen:
   180	                continue
   181	            seen.add(key_s)
   182	            parsed = self._to_number(value)
   183	            if parsed is not None:
   184	                numeric[key_s] = parsed
   185	        return numeric
   186
   187	    def _apply_stats(self, perf: Dict[str, Any], prefix: str, values: List[float]) -> None:
   188	        if not values:
   189	            return
   190	        perf[f"{prefix}_total"] = float(sum(values))
   191	        perf[f"{prefix}_min"] = float(min(values))
   192	        perf[f"{prefix}_max"] = float(max(values))
   193	        perf[f"{prefix}_p50"] = float(self._p50(values))
   194
   195	    def _get_decoder_pending_requests(self) -> int:
   196	        getter = getattr(self.snac_decoder, "get_pending_requests", None)
   197	        if not callable(getter):
   198	            return 0
   199	        try:
   200	            return max(0, int(getter()))
   201	        except Exception:
   202	            return 0
   203
   204	    def _resolve_decode_interval(self, first_chunk_emitted: bool) -> Tuple[int, int]:
   205	        interval = int(_STREAM_DECODE_INTERVAL)
   206	        pending = 0
   207	        if not _STREAM_ADAPTIVE_DECODE:
   208	            return interval, pending
   209
   210	        if not first_chunk_emitted:
   211	            interval = int(_STREAM_DECODE_INTERVAL_FIRST)
   212
   213	        pending = self._get_decoder_pending_requests()
   214	        if pending >= _STREAM_DECODE_BUSY_PENDING:
   215	            interval = max(interval, int(_STREAM_DECODE_INTERVAL_BUSY))
   216	        return max(1, interval), pending
   217
   218	    async def generate_speech_stream(
   219	        self,
   220	        description: str,
   221	        text: str,
   222	        temperature: float = DEFAULT_TEMPERATURE,
   223	        top_p: float = DEFAULT_TOP_P,
   224	        max_tokens: int = DEFAULT_MAX_TOKENS,
   225	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   226	        seed: Optional[int] = None,
   227	    ) -> AsyncGenerator[bytes, None]:
   228	        """
   229	        Generate speech audio with sliding window streaming.
   230
   231	        Yields audio chunks using overlapping windows for smooth playback.
   232
   233	        Args:
   234	            description: Character/voice description
   235	            text: Text to synthesize (with optional <emotion> tags)


Output:
Chunk ID: 627b4f
Wall time: 0.0502 seconds
Process exited with code 0
Original token count: 1890
Output:
   500	                elif "model_forward_time" in raw_req_metrics:
   501	                    cumulative_gpu_s = raw_req_metrics["model_forward_time"]
   502
   503	                if cumulative_gpu_s is not None:
   504	                    if prev_gpu_cumulative_s is None:
   505	                        gpu_delta_ms = cumulative_gpu_s * 1000.0
   506	                    elif cumulative_gpu_s >= prev_gpu_cumulative_s:
   507	                        gpu_delta_ms = (cumulative_gpu_s - prev_gpu_cumulative_s) * 1000.0
   508	                    else:
   509	                        gpu_delta_ms = cumulative_gpu_s * 1000.0
   510	                    prev_gpu_cumulative_s = cumulative_gpu_s
   511	                    if gpu_delta_ms >= 0.0:
   512	                        llm_batch_gpu_ms.append(float(gpu_delta_ms))
   513	                        if len(llm_batch_wall_ms) > 1:
   514	                            llm_decode_gpu_ms.append(float(gpu_delta_ms))
   515
   516	            parse_t0 = time.perf_counter()
   517	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
   518	            llm_parse_ms_samples.append((time.perf_counter() - parse_t0) * 1000.0)
   519
   520	            if len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
   521	                semantic_count = len(semantic_buffer)
   522	                if semantic_count >= _STREAM_MIN_SEMANTIC_FIRST_CHUNK:
   523	                    if "first_chunk_ready" not in mark_stage_set:
   524	                        _mark("first_chunk_ready")
   525
   526	                    decode_interval, pending_requests = self._resolve_decode_interval(
   527	                        first_chunk_emitted=t_first_chunk_emitted is not None
   528	                    )
   529	                    semantic_delta = semantic_count - last_decode_count
   530	                    if semantic_delta >= decode_interval:
   531	                        decode_step_tokens = semantic_delta
   532	                        last_decode_count = semantic_count
   533	                        decode_interval_applied_tokens.append(float(decode_interval))
   534	                        decode_pending_requests.append(float(pending_requests))
   535	                        decode_global = global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT]
   536
   537	                        if (
   538	                            _STREAM_WINDOWED_DECODE
   539	                            and semantic_count > _STREAM_WINDOW_SIZE
   540	                        ):
   541	                            decode_semantic = semantic_buffer[-_STREAM_WINDOW_SIZE:]
   542	                        else:
   543	                            decode_semantic = semantic_buffer
   544
   545	                        decode_t0 = time.perf_counter()
   546	                        audio_bytes = await self.snac_decoder.decode_single_async(
   547	                            semantic_ids=decode_semantic,
   548	                            global_ids=decode_global,
   549	                            trim_warmup=False,
   550	                            use_sliding_window=False,
   551	                        )
   552	                        decode_t1 = time.perf_counter()
   553
   554	                        decode_wall_ms = (decode_t1 - decode_t0) * 1000.0
   555	                        bicodec_decode_wall_ms.append(float(decode_wall_ms))
   556	                        bicodec_decode_tokens.append(float(len(decode_semantic)))
   557	                        if last_decode_end_ts is not None:
   558	                            bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
   559	                        last_decode_end_ts = decode_t1
   560
   561	                        if audio_bytes:
   562	                            total_samples_decoded = len(audio_bytes) // 2
   563	                            if (
   564	                                _STREAM_WINDOWED_DECODE
   565	                                and semantic_count > _STREAM_WINDOW_SIZE
   566	                            ):
   567	                                new_samples_approx = decode_step_tokens * 320
   568	                                if total_samples_decoded > new_samples_approx:
   569	                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
   570	                                else:
   571	                                    new_bytes_start = 0
   572	                                new_audio_bytes = audio_bytes[new_bytes_start:]
   573	                            else:
   574	                                if total_samples_decoded > total_samples_emitted_to_user:
   575	                                    new_bytes_start = total_samples_emitted_to_user * 2
   576	                                    new_audio_bytes = audio_bytes[new_bytes_start:]
   577	                                else:
   578	                                    new_audio_bytes = b""
   579
   580	                            if new_audio_bytes:
   581	                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   582	                                    previous_chunk_tail,
   583	                                    new_audio_bytes,
   584	                                    sample_rate_hz=16000,
   585	                                    crossfade_ms=_STREAM_CROSSFADE_MS,
   586	                                )
   587	                                if to_emit:
   588	                                    total_audio_chunks += 1
   589	                                    total_samples_emitted_to_user += len(to_emit) // 2
   590	                                    if t_first_chunk_decode_done is None:
   591	                                        t_first_chunk_decode_done = decode_t1
   592	                                        _mark("first_chunk_decoded")
   593	                                    if t_first_chunk_emitted is None:
   594	                                        t_first_chunk_emitted = time.perf_counter()
   595	                                        _mark("first_chunk_emitted")
   596	                                    if emit_progress:
   597	                                        yield (to_emit, {})
   598	                                    else:
   599	                                        yield to_emit
   600
   601	        llm_generation_wall_ms = (time.perf_counter() - t_llm_start) * 1000.0
   602	        _mark("llm_done")
   603
   604	        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
   605	            decode_semantic = semantic_buffer
   606	            decode_t0 = time.perf_counter()
   607	            audio_bytes = self.snac_decoder.decode_streaming(
   608	                semantic_ids=decode_semantic,
   609	                global_ids=global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT],
   610	                use_sliding_window=False,
   611	                trim_warmup=False,
   612	            )
   613	            decode_t1 = time.perf_counter()
   614	            decode_wall_ms = (decode_t1 - decode_t0) * 1000.0
   615	            bicodec_decode_wall_ms.append(float(decode_wall_ms))
   616	            bicodec_decode_tokens.append(float(len(decode_semantic)))
   617	            if last_decode_end_ts is not None:
   618	                bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
   619	            last_decode_end_ts = decode_t1
   620


Output:
Chunk ID: 3430f2
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 1023
Output:
   660	        perf: Dict[str, Any] = {
   661	            "prompt_build_ms": float(prompt_build_ms),
   662	            "sampling_params_ms": float(sampling_params_ms),
   663	            "llm_generation_wall_ms": float(llm_generation_wall_ms),
   664	            "llm_batch_count": int(len(llm_batch_wall_ms)),
   665	            "llm_token_total": int(total_generated_tokens),
   666	            "semantic_token_total": int(len(semantic_buffer)),
   667	            "global_token_total": int(len(global_buffer)),
   668	            "llm_parse_ms": float(sum(llm_parse_ms_samples)),
   669	            "bicodec_decode_calls": int(len(bicodec_decode_wall_ms)),
   670	            "llm_decode_calls": int(len(bicodec_decode_wall_ms)),
   671	            "chunks_sent": int(total_audio_chunks),
   672	            "audio_duration_seconds": float(total_samples_emitted_to_user / 16000.0),
   673	            "ttfb_ms": float((t_first_chunk_emitted - t_request_start) * 1000.0 if t_first_chunk_emitted else 0.0),
   674	            "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
   675	            "stream_decode_interval_tokens": int(_STREAM_DECODE_INTERVAL),
   676	            "stream_decode_interval_first_tokens": int(_STREAM_DECODE_INTERVAL_FIRST),
   677	            "stream_decode_interval_busy_tokens": int(_STREAM_DECODE_INTERVAL_BUSY),
   678	            "stream_decode_busy_pending_threshold": int(_STREAM_DECODE_BUSY_PENDING),
   679	            "stream_adaptive_decode": bool(_STREAM_ADAPTIVE_DECODE),
   680	            "stream_window_size_tokens": int(_STREAM_WINDOW_SIZE),
   681	            "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
   682	            "timeline_markers": list(timeline_marks),
   683	            "timeline_total_ms": float((time.perf_counter() - t_request_start) * 1000.0),
   684	        }
   685
   686	        decoder_batch_stats = {}
   687	        if hasattr(self.snac_decoder, "get_batching_stats"):
   688	            try:
   689	                decoder_batch_stats = self.snac_decoder.get_batching_stats() or {}
   690	            except Exception:
   691	                decoder_batch_stats = {}
   692	        if decoder_batch_stats:
   693	            perf.update(decoder_batch_stats)
   694
   695	        self._apply_stats(perf, "llm_batch_wall_ms", llm_batch_wall_ms)
   696	        self._apply_stats(perf, "llm_batch_gpu_ms", llm_batch_gpu_ms)
   697	        self._apply_stats(perf, "llm_decode_wall_ms", llm_decode_wall_ms)
   698	        self._apply_stats(perf, "llm_decode_gpu_ms", llm_decode_gpu_ms)
   699	        self._apply_stats(perf, "tokens_per_batch", tokens_per_batch)
   700	        self._apply_stats(perf, "llm_parse_step_ms", llm_parse_ms_samples)
   701	        self._apply_stats(perf, "bicodec_decode_wall_ms", bicodec_decode_wall_ms)
   702	        self._apply_stats(perf, "bicodec_decode_tokens", bicodec_decode_tokens)
   703	        self._apply_stats(perf, "bicodec_decode_interval_ms", bicodec_decode_intervals_ms)
   704	        self._apply_stats(perf, "decode_interval_applied_tokens", decode_interval_applied_tokens)
   705	        self._apply_stats(perf, "decode_pending_requests", decode_pending_requests)
   706
   707	        perf["bicodec_decode_wall_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))
   708	        perf["bicodec_decode_gpu_ms"] = 0.0
   709	        perf["bicodec_decode_cpu_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))
   710
   711	        if total_generated_tokens > 0:
   712	            perf["llm_time_per_token_ms"] = float(llm_generation_wall_ms) / float(total_generated_tokens)
   713	        if llm_batch_wall_ms:
   714	            perf["llm_time_per_batch_wall_ms"] = float(sum(llm_batch_wall_ms)) / float(len(llm_batch_wall_ms))
   715	        if llm_batch_gpu_ms:
   716	            perf["llm_time_per_batch_gpu_ms"] = float(sum(llm_batch_gpu_ms)) / float(len(llm_batch_gpu_ms))
   717
   718	        for mark in timeline_marks:
   719	            stage = mark.get("stage")
   720	            if not stage:


Output:
Chunk ID: 473a53
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 1564
Output:
  1050	        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)
  1051
  1052	        # Configure sampling
  1053	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
  1054	        sampling_params = SamplingParams(
  1055	            temperature=temperature,
  1056	            top_k=top_k,
  1057	            top_p=top_p,
  1058	            max_tokens=max_tokens,
  1059	            min_tokens=DEFAULT_MIN_TOKENS,
  1060	            repetition_penalty=repetition_penalty,
  1061	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
  1062	            skip_special_tokens=False,
  1063	            seed=seed,
  1064	            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
  1065	        )
  1066
  1067	        # BiCodec token buffers
  1068	        semantic_buffer = []
  1069	        global_buffer_unused = []  # Not used in continuation, but needed for parser API
  1070	        processed_token_count = 0  # Used for cumulative mode; ignored for delta mode.
  1071	        total_audio_chunks = 0
  1072	        total_generated_tokens = 0
  1073
  1074	        # BiCodec streaming configuration
  1075	        MIN_SEMANTIC_FOR_FIRST_CHUNK = _STREAM_MIN_SEMANTIC_FIRST_CHUNK
  1076	        DECODE_INTERVAL = _STREAM_DECODE_INTERVAL
  1077	        CROSSFADE_MS = _STREAM_CROSSFADE_MS
  1078	        WINDOW_SIZE = _STREAM_WINDOW_SIZE
  1079
  1080	        # Generate unique request ID
  1081	        import uuid
  1082	        request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
  1083
  1084	        # Reuse pre-warmed token parser (singleton, created once in __init__)
  1085	        token_parser = self.token_parser
  1086
  1087	        results_generator = self.model.engine.generate(
  1088	            prompt=prompt,
  1089	            sampling_params=sampling_params,
  1090	            request_id=request_id,
  1091	        )
  1092
  1093	        # Track state
  1094	        last_decode_count = 0
  1095	        total_samples_emitted_to_user = 0
  1096	        previous_chunk_tail: Optional[bytes] = None
  1097	        t_first_chunk_emitted: Optional[float] = None
  1098	        decode_calls = 0
  1099	        decode_wall_ms_total = 0.0
  1100	        decode_interval_applied_tokens: List[float] = []
  1101	        decode_pending_requests: List[float] = []
  1102
  1103	        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
  1104	        async for request_output in results_generator:
  1105	            generated_ids = request_output.outputs[0].token_ids
  1106
  1107	            # In DELTA mode token_ids are already incremental.
  1108	            if _STREAM_USE_DELTA_OUTPUT:
  1109	                new_token_ids = generated_ids
  1110	                processed_token_count += len(new_token_ids)
  1111	            else:
  1112	                new_token_ids = generated_ids[processed_token_count:]
  1113	                processed_token_count = len(generated_ids)
  1114	            total_generated_tokens += len(new_token_ids)
  1115	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)
  1116
  1117	            semantic_count = len(semantic_buffer)
  1118	            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
  1119	                decode_interval, pending_requests = self._resolve_decode_interval(
  1120	                    first_chunk_emitted=t_first_chunk_emitted is not None
  1121	                )
  1122	                semantic_delta = semantic_count - last_decode_count
  1123	                if semantic_delta >= decode_interval:
  1124	                    decode_step_tokens = semantic_delta
  1125	                    last_decode_count = semantic_count
  1126	                    decode_interval_applied_tokens.append(float(decode_interval))
  1127	                    decode_pending_requests.append(float(pending_requests))
  1128	                    if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
  1129	                        decode_semantic = semantic_buffer[-WINDOW_SIZE:]
  1130	                    else:
  1131	                        decode_semantic = semantic_buffer
  1132
  1133	                    # Use pre-captured global tokens for decoding
  1134	                    decode_t0 = time.perf_counter()
  1135	                    if self.snac_decoder.enable_batching:
  1136	                        audio_bytes = await self.snac_decoder.decode_single_async(
  1137	                            semantic_ids=decode_semantic,
  1138	                            global_ids=global_ids,  # Use captured globals
  1139	                            trim_warmup=False,
  1140	                            use_sliding_window=False
  1141	                        )
  1142	                    else:
  1143	                        audio_bytes = self.snac_decoder.decode_streaming(
  1144	                            semantic_ids=decode_semantic,
  1145	                            global_ids=global_ids,  # Use captured globals
  1146	                            use_sliding_window=False,
  1147	                            trim_warmup=False
  1148	                        )
  1149	                    decode_calls += 1
  1150	                    decode_wall_ms_total += (time.perf_counter() - decode_t0) * 1000.0
  1151
  1152	                    if audio_bytes:
  1153	                        total_samples_decoded = len(audio_bytes) // 2
  1154	                        if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
  1155	                            new_samples_approx = decode_step_tokens * 320
  1156	                            if total_samples_decoded > new_samples_approx:
  1157	                                new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
  1158	                            else:
  1159	                                new_bytes_start = 0
  1160	                            new_audio_bytes = audio_bytes[new_bytes_start:]


Output:
Chunk ID: d42656
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 568
Output:
  1240	        if emit_progress:
  1241	            perf: Dict[str, Any] = {
  1242	                "llm_token_total": int(total_generated_tokens),
  1243	                "semantic_token_total": int(len(semantic_buffer)),
  1244	                "global_token_total": int(len(global_ids)),
  1245	                "bicodec_decode_calls": int(decode_calls),
  1246	                "llm_decode_calls": int(decode_calls),
  1247	                "bicodec_decode_wall_ms": float(decode_wall_ms_total),
  1248	                "bicodec_decode_gpu_ms": 0.0,
  1249	                "bicodec_decode_cpu_ms": float(decode_wall_ms_total),
  1250	                "chunks_sent": int(total_audio_chunks),
  1251	                "audio_duration_seconds": float(audio_duration_s),
  1252	                "ttfb_ms": float((t_first_chunk_emitted - t_start) * 1000.0 if t_first_chunk_emitted else 0.0),
  1253	                "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
  1254	                "stream_decode_interval_tokens": int(DECODE_INTERVAL),
  1255	                "stream_decode_interval_first_tokens": int(_STREAM_DECODE_INTERVAL_FIRST),
  1256	                "stream_decode_interval_busy_tokens": int(_STREAM_DECODE_INTERVAL_BUSY),
  1257	                "stream_decode_busy_pending_threshold": int(_STREAM_DECODE_BUSY_PENDING),
  1258	                "stream_adaptive_decode": bool(_STREAM_ADAPTIVE_DECODE),
  1259	                "stream_window_size_tokens": int(WINDOW_SIZE),
  1260	                "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
  1261	            }
  1262	            self._apply_stats(perf, "decode_interval_applied_tokens", decode_interval_applied_tokens)
  1263	            self._apply_stats(perf, "decode_pending_requests", decode_pending_requests)
  1264	            decoder_batch_stats = {}
  1265	            if hasattr(self.snac_decoder, "get_batching_stats"):
  1266	                try:
  1267	                    decoder_batch_stats = self.snac_decoder.get_batching_stats() or {}
  1268	                except Exception:
  1269	                    decoder_batch_stats = {}
  1270	            if decoder_batch_stats:
  1271	                perf.update(decoder_batch_stats)
  1272	            yield (b"", perf)


Summary: auto


**Adding line ranges for metrics**


nl -ba scripts/stress_test_streaming_runtime.py | sed -n '230,340p'


nl -ba scripts/stress_test_streaming_runtime.py | sed -n '360,470p'


nl -ba scripts/stress_test_streaming_runtime.py | sed -n '500,620p'


nl -ba scripts/stress_test_streaming_runtime.py | sed -n '620,760p'


Output:
Chunk ID: a50e33
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1195
Output:
   230	    from veena3modal.local_server import resolve_model_paths
   231	    from veena3modal.services.tts_runtime import initialize_runtime, is_initialized
   232
   233	    os.environ["VEENA3_STREAM_OUTPUT_KIND"] = stream_output_kind
   234
   235	    llm_path, bicodec_path = resolve_model_paths(model_path)
   236	    if not is_initialized():
   237	        initialize_runtime(
   238	            model_path=llm_path,
   239	            bicodec_path=bicodec_path,
   240	            device="cuda",
   241	            gpu_memory_utilization=gpu_mem,
   242	            num_engines=num_engines,
   243	            max_num_batched_tokens=max_num_batched_tokens,
   244	            max_num_seqs=max_num_seqs,
   245	            enable_chunked_prefill=enable_chunked_prefill,
   246	            enable_prefix_caching=enable_prefix_caching,
   247	            disable_log_stats=disable_log_stats,
   248	            enforce_eager=enforce_eager,
   249	            precompute_speaker_globals=False,
   250	        )
   251
   252
   253	async def run_one_request(
   254	    text: str,
   255	    speaker: str,
   256	    max_tokens: int,
   257	    timeout_s: float,
   258	    enable_chunking: bool,
   259	    seed: Optional[int],
   260	) -> RequestResult:
   261	    from veena3modal.services import tts_runtime
   262
   263	    t0 = time.perf_counter()
   264	    first_chunk_t: Optional[float] = None
   265	    prev_chunk_t: Optional[float] = None
   266	    inter_chunk_ms: List[float] = []
   267	    chunks = 0
   268	    total_bytes = 0
   269	    final_metrics: Dict[str, Any] = {}
   270
   271	    async def _consume_stream() -> None:
   272	        nonlocal first_chunk_t, prev_chunk_t, chunks, total_bytes, final_metrics
   273	        gen = tts_runtime.generate_speech_streaming(
   274	            text=text,
   275	            speaker=speaker,
   276	            max_tokens=max_tokens,
   277	            enable_chunking=enable_chunking,
   278	            seed=seed,
   279	        )
   280	        async for audio_chunk, metrics in gen:
   281	            now = time.perf_counter()
   282	            if first_chunk_t is None:
   283	                first_chunk_t = now
   284	            if prev_chunk_t is not None:
   285	                inter_chunk_ms.append((now - prev_chunk_t) * 1000.0)
   286	            prev_chunk_t = now
   287	            chunks += 1
   288	            total_bytes += len(audio_chunk)
   289	            if isinstance(metrics, dict):
   290	                # Keep reference; runtime may mutate this dict after the final chunk
   291	                # when internal stream generation completes.
   292	                final_metrics = metrics
   293
   294	    try:
   295	        await asyncio.wait_for(_consume_stream(), timeout=timeout_s)
   296	        end = time.perf_counter()
   297	        latency_ms = (end - t0) * 1000.0
   298	        ttfb_ms = ((first_chunk_t - t0) * 1000.0) if first_chunk_t is not None else 0.0
   299	        audio_seconds = float(final_metrics.get("audio_duration_seconds", 0.0) or 0.0)
   300	        tail_s = max(1e-6, (latency_ms - ttfb_ms) / 1000.0)
   301	        tail_emit_speed_x = audio_seconds / tail_s if audio_seconds > 0 else 0.0
   302	        return RequestResult(
   303	            success=True,
   304	            latency_ms=latency_ms,
   305	            ttfb_ms=ttfb_ms,
   306	            chunks_sent=chunks,
   307	            total_bytes=total_bytes,
   308	            audio_seconds=audio_seconds,
   309	            speaker=speaker,
   310	            inter_chunk_ms=inter_chunk_ms,
   311	            tail_emit_speed_x=tail_emit_speed_x,
   312	            final_metrics=dict(final_metrics),
   313	        )
   314	    except asyncio.TimeoutError:
   315	        return RequestResult(
   316	            success=False,
   317	            latency_ms=(time.perf_counter() - t0) * 1000.0,
   318	            ttfb_ms=0.0,
   319	            chunks_sent=chunks,
   320	            total_bytes=total_bytes,
   321	            audio_seconds=0.0,
   322	            speaker=speaker,
   323	            error="timeout",
   324	            inter_chunk_ms=inter_chunk_ms,
   325	            final_metrics=dict(final_metrics),
   326	        )
   327	    except Exception as exc:
   328	        return RequestResult(
   329	            success=False,
   330	            latency_ms=(time.perf_counter() - t0) * 1000.0,
   331	            ttfb_ms=0.0,
   332	            chunks_sent=chunks,
   333	            total_bytes=total_bytes,
   334	            audio_seconds=0.0,
   335	            speaker=speaker,
   336	            error=str(exc)[:200],
   337	            inter_chunk_ms=inter_chunk_ms,
   338	            final_metrics=dict(final_metrics),
   339	        )
   340


Output:
Chunk ID: 184d6f
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 1264
Output:
   360	                text=text,
   361	                speaker=speaker,
   362	                max_tokens=max_tokens,
   363	                timeout_s=timeout_s,
   364	                enable_chunking=enable_chunking,
   365	                seed=seed,
   366	            )
   367
   368	    gpu_monitor.snapshots.clear()
   369	    t0 = time.perf_counter()
   370	    level.results = await asyncio.gather(*[wrapped(i) for i in range(total_requests)])
   371	    level.wall_time_s = time.perf_counter() - t0
   372	    level.gpu_summary = gpu_monitor.summary()
   373	    return level
   374
   375
   376	def print_level(level: LevelResult) -> None:
   377	    print(f"\n=== streaming concurrency={level.concurrency} requests={level.total_requests} ===")
   378	    print(f"success={len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
   379	    print(f"wall={level.wall_time_s:.2f}s throughput={level.throughput_rps:.2f} req/s")
   380	    lat = level.latency_stats()
   381	    if lat:
   382	        print(f"latency_ms avg={lat['avg']:.0f} p50={lat['p50']:.0f} p95={lat['p95']:.0f} max={lat['max']:.0f}")
   383	    ttfb = level.ttfb_stats()
   384	    if ttfb:
   385	        print(f"ttfb_ms avg={ttfb['avg']:.0f} p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}")
   386	    cc = level.chunk_count_stats()
   387	    if cc:
   388	        print(f"chunks_per_req avg={cc['avg']:.1f} p50={cc['p50']:.1f} p95={cc['p95']:.1f} max={cc['max']:.1f}")
   389	    ic = level.inter_chunk_stats()
   390	    if ic:
   391	        print(
   392	            "inter_chunk_ms avg={:.1f} p50={:.1f} p95={:.1f} max={:.1f}".format(
   393	                ic["avg"], ic["p50"], ic["p95"], ic["max"]
   394	            )
   395	        )
   396	    es = level.emit_speed_stats()
   397	    if es:
   398	        print(
   399	            "tail_emit_speed_x avg={:.2f} p50={:.2f} p95={:.2f} max={:.2f}".format(
   400	                es["avg"], es["p50"], es["p95"], es["max"]
   401	            )
   402	        )
   403
   404	    timing_keys = [
   405	        "timeline_to_first_batch_ms",
   406	        "timeline_to_first_chunk_emitted_ms",
   407	        "timeline_total_ms",
   408	        "llm_time_in_queue_ms",
   409	        "llm_scheduler_ms",
   410	        "llm_first_token_ms",
   411	        "llm_request_lifecycle_ms",
   412	        "llm_generation_wall_ms",
   413	        "llm_time_per_token_ms",
   414	        "llm_batch_wall_ms_total",
   415	        "llm_decode_wall_ms_total",
   416	        "tokens_per_batch_p50",
   417	        "bicodec_decode_calls",
   418	        "bicodec_decode_wall_ms_total",
   419	        "bicodec_decode_interval_ms_p50",
   420	        "decode_interval_applied_tokens_p50",
   421	        "decode_pending_requests_p50",
   422	        "batch_avg_size",
   423	        "batch_max_seen",
   424	        "batch_queue_wait_ms_avg",
   425	        "batch_compute_ms_avg",
   426	        "batch_queue_depth_avg",
   427	    ]
   428	    for key in timing_keys:
   429	        st = level.timing_stats(key)
   430	        if st:
   431	            print(f"{key}: avg={st['avg']:.2f} p50={st['p50']:.2f} p95={st['p95']:.2f} max={st['max']:.2f}")
   432
   433	    latency_over_timeline = level.latency_over_metric_stats("timeline_total_ms")
   434	    if latency_over_timeline:
   435	        print(
   436	            "latency_over_timeline_ms: avg={:.2f} p50={:.2f} p95={:.2f} max={:.2f}".format(
   437	                latency_over_timeline["avg"],
   438	                latency_over_timeline["p50"],
   439	                latency_over_timeline["p95"],
   440	                latency_over_timeline["max"],
   441	            )
   442	        )
   443
   444	    tl_total = level.timing_stats("timeline_total_ms")
   445	    llm_total = level.timing_stats("llm_generation_wall_ms")
   446	    bicodec_total = level.timing_stats("bicodec_decode_wall_ms_total")
   447	    if tl_total and llm_total:
   448	        total_avg = tl_total.get("avg", 0.0)
   449	        llm_avg = llm_total.get("avg", 0.0)
   450	        bicodec_avg = bicodec_total.get("avg", 0.0) if bicodec_total else 0.0
   451	        other_avg = max(0.0, total_avg - llm_avg - bicodec_avg)
   452	        if total_avg > 0:
   453	            print(
   454	                "timeline_share: llm={:.1f}% bicodec={:.1f}% other={:.1f}%".format(
   455	                    (llm_avg / total_avg) * 100,
   456	                    (bicodec_avg / total_avg) * 100,
   457	                    (other_avg / total_avg) * 100,
   458	                )
   459	            )
   460
   461	    if "memory_mb" in level.gpu_summary:
   462	        g = level.gpu_summary
   463	        print(
   464	            "gpu: util_avg={:.0f}% util_max={:.0f}% mem_avg={:.0f}MB mem_max={:.0f}MB".format(
   465	                g["gpu_util_pct"]["avg"],
   466	                g["gpu_util_pct"]["max"],
   467	                g["memory_mb"]["avg"],
   468	                g["memory_mb"]["max"],
   469	            )
   470	        )


Output:
Chunk ID: 21d5f3
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 1786
Output:
   500	            "adaptive_decode_busy_interval": args.adaptive_decode_busy_interval,
   501	            "adaptive_decode_busy_pending": args.adaptive_decode_busy_pending,
   502	            "disable_windowed_decode": args.disable_windowed_decode,
   503	            "disable_bicodec_batching": args.disable_bicodec_batching,
   504	            "bicodec_batch_max": args.bicodec_batch_max,
   505	            "bicodec_batch_timeout_ms": args.bicodec_batch_timeout_ms,
   506	            "timestamp": int(time.time()),
   507	        },
   508	        "levels": [],
   509	    }
   510	    for lvl in levels:
   511	        payload["levels"].append(
   512	            {
   513	                "concurrency": lvl.concurrency,
   514	                "total_requests": lvl.total_requests,
   515	                "success_rate": lvl.success_rate,
   516	                "throughput_rps": lvl.throughput_rps,
   517	                "wall_time_s": lvl.wall_time_s,
   518	                "latency_ms": lvl.latency_stats(),
   519	                "ttfb_ms": lvl.ttfb_stats(),
   520	                "chunks_per_req": lvl.chunk_count_stats(),
   521	                "inter_chunk_ms": lvl.inter_chunk_stats(),
   522	                "tail_emit_speed_x": lvl.emit_speed_stats(),
   523	                "gpu": lvl.gpu_summary,
   524	                "timing": {
   525	                    "timeline_to_first_batch_ms": lvl.timing_stats("timeline_to_first_batch_ms"),
   526	                    "timeline_to_first_chunk_emitted_ms": lvl.timing_stats("timeline_to_first_chunk_emitted_ms"),
   527	                    "timeline_total_ms": lvl.timing_stats("timeline_total_ms"),
   528	                    "llm_time_in_queue_ms": lvl.timing_stats("llm_time_in_queue_ms"),
   529	                    "llm_scheduler_ms": lvl.timing_stats("llm_scheduler_ms"),
   530	                    "llm_first_token_ms": lvl.timing_stats("llm_first_token_ms"),
   531	                    "llm_request_lifecycle_ms": lvl.timing_stats("llm_request_lifecycle_ms"),
   532	                    "llm_generation_wall_ms": lvl.timing_stats("llm_generation_wall_ms"),
   533	                    "llm_time_per_token_ms": lvl.timing_stats("llm_time_per_token_ms"),
   534	                    "llm_batch_wall_ms_total": lvl.timing_stats("llm_batch_wall_ms_total"),
   535	                    "llm_decode_wall_ms_total": lvl.timing_stats("llm_decode_wall_ms_total"),
   536	                    "tokens_per_batch_p50": lvl.timing_stats("tokens_per_batch_p50"),
   537	                    "bicodec_decode_calls": lvl.timing_stats("bicodec_decode_calls"),
   538	                    "bicodec_decode_wall_ms_total": lvl.timing_stats("bicodec_decode_wall_ms_total"),
   539	                    "bicodec_decode_interval_ms_p50": lvl.timing_stats("bicodec_decode_interval_ms_p50"),
   540	                    "decode_interval_applied_tokens_p50": lvl.timing_stats("decode_interval_applied_tokens_p50"),
   541	                    "decode_pending_requests_p50": lvl.timing_stats("decode_pending_requests_p50"),
   542	                    "batch_avg_size": lvl.timing_stats("batch_avg_size"),
   543	                    "batch_max_seen": lvl.timing_stats("batch_max_seen"),
   544	                    "batch_queue_wait_ms_avg": lvl.timing_stats("batch_queue_wait_ms_avg"),
   545	                    "batch_compute_ms_avg": lvl.timing_stats("batch_compute_ms_avg"),
   546	                    "batch_queue_depth_avg": lvl.timing_stats("batch_queue_depth_avg"),
   547	                    "latency_over_timeline_ms": lvl.latency_over_metric_stats("timeline_total_ms"),
   548	                },
   549	                "requests": [
   550	                    {
   551	                        "success": r.success,
   552	                        "latency_ms": r.latency_ms,
   553	                        "ttfb_ms": r.ttfb_ms,
   554	                        "chunks_sent": r.chunks_sent,
   555	                        "total_bytes": r.total_bytes,
   556	                        "audio_seconds": r.audio_seconds,
   557	                        "speaker": r.speaker,
   558	                        "error": r.error,
   559	                        "inter_chunk_ms": r.inter_chunk_ms,
   560	                        "tail_emit_speed_x": r.tail_emit_speed_x,
   561	                        "final_metrics": r.final_metrics,
   562	                    }
   563	                    for r in lvl.results
   564	                ],
   565	            }
   566	        )
   567	    with open(path, "w", encoding="utf-8") as fd:
   568	        json.dump(payload, fd, indent=2)
   569
   570
   571	async def main_async(args: argparse.Namespace) -> None:
   572	    text = TEST_TEXTS[args.text]
   573	    levels = [int(x.strip()) for x in args.levels.split(",") if x.strip()]
   574	    gpu_monitor = GPUMonitor(interval_seconds=0.5)
   575	    gpu_monitor.start()
   576
   577	    # warmup
   578	    await run_one_request(
   579	        text=text,
   580	        speaker=SPEAKERS[0],
   581	        max_tokens=args.max_tokens,
   582	        timeout_s=args.timeout,
   583	        enable_chunking=args.chunking,
   584	        seed=args.seed_base,
   585	    )
   586
   587	    results: List[LevelResult] = []
   588	    for idx, c in enumerate(levels):
   589	        n = max(args.min_requests, c * args.requests_multiplier)
   590	        level = await run_level(
   591	            concurrency=c,
   592	            total_requests=n,
   593	            text=text,
   594	            max_tokens=args.max_tokens,
   595	            timeout_s=args.timeout,
   596	            enable_chunking=args.chunking,
   597	            gpu_monitor=gpu_monitor,
   598	            seed_base=args.seed_base,
   599	        )
   600	        print_level(level)
   601	        results.append(level)
   602	        if idx < len(levels) - 1:
   603	            await asyncio.sleep(2)
   604
   605	    gpu_monitor.stop()
   606	    save_results(args.output, results, args)
   607	    print(f"\nresults saved: {args.output}")
   608
   609
   610	def main() -> None:
   611	    parser = argparse.ArgumentParser(description="Direct runtime streaming stress benchmark")
   612	    parser.add_argument("--levels", default="16,32,64", help="Comma-separated concurrency levels")
   613	    parser.add_argument("--text", choices=["short", "medium", "long"], default="short")
   614	    parser.add_argument("--max-tokens", type=int, default=512, help="max_tokens sent to streaming generation")
   615	    parser.add_argument("--seed-base", type=int, default=1337, help="Base random seed; request i uses seed_base+i")
   616	    parser.add_argument("--timeout", type=float, default=180.0, help="Per-request timeout seconds")
   617	    parser.add_argument("--requests-multiplier", type=int, default=2, help="Requests per level ~= concurrency * multiplier")
   618	    parser.add_argument("--min-requests", type=int, default=8)
   619	    parser.add_argument("--chunking", action="store_true", default=False, help="Enable long-text chunking path")
   620	    parser.add_argument("--gpu-memory", type=float, default=0.25)


Output:
Chunk ID: 992f35
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 1101
Output:
   620	    parser.add_argument("--gpu-memory", type=float, default=0.25)
   621	    parser.add_argument("--stream-output-kind", choices=["delta", "cumulative"], default="delta")
   622	    parser.add_argument("--num-engines", type=int, default=1, help="Number of vLLM engines")
   623	    parser.add_argument("--max-num-batched-tokens", type=int, default=None)
   624	    parser.add_argument("--max-num-seqs", type=int, default=None)
   625	    parser.add_argument("--enable-chunked-prefill", action="store_true", default=None)
   626	    parser.add_argument("--disable-chunked-prefill", action="store_true", default=False)
   627	    parser.add_argument("--disable-prefix-caching", action="store_true", default=False)
   628	    parser.add_argument("--disable-engine-stats-logs", action="store_true", default=False)
   629	    parser.add_argument("--enforce-eager", action="store_true", default=False)
   630	    parser.add_argument("--stream-decode-interval", type=int, default=48)
   631	    parser.add_argument("--stream-window-size", type=int, default=128)
   632	    parser.add_argument("--stream-min-semantic-first", type=int, default=10)
   633	    parser.add_argument("--stream-crossfade-ms", type=int, default=50)
   634	    parser.add_argument("--adaptive-decode", action="store_true", default=False)
   635	    parser.add_argument("--adaptive-decode-first-interval", type=int, default=48)
   636	    parser.add_argument("--adaptive-decode-busy-interval", type=int, default=64)
   637	    parser.add_argument("--adaptive-decode-busy-pending", type=int, default=32)
   638	    parser.add_argument("--disable-windowed-decode", action="store_true", default=False)
   639	    parser.add_argument("--disable-bicodec-batching", action="store_true", default=False)
   640	    parser.add_argument("--bicodec-batch-max", type=int, default=16)
   641	    parser.add_argument("--bicodec-batch-timeout-ms", type=float, default=6.0)
   642	    parser.add_argument("--model-path", type=str, default="")
   643	    parser.add_argument("--output", default="stress_streaming_runtime.json")
   644	    args = parser.parse_args()
   645
   646	    from veena3modal.local_server import DEFAULT_LOCAL_MODEL_DIR
   647
   648	    model_path = args.model_path or DEFAULT_LOCAL_MODEL_DIR
   649
   650	    os.environ["VEENA3_STREAM_OUTPUT_KIND"] = args.stream_output_kind
   651	    os.environ["VEENA3_STREAM_DECODE_INTERVAL"] = str(args.stream_decode_interval)
   652	    os.environ["VEENA3_STREAM_WINDOW_SIZE"] = str(args.stream_window_size)
   653	    os.environ["VEENA3_STREAM_MIN_SEMANTIC_FIRST"] = str(args.stream_min_semantic_first)
   654	    os.environ["VEENA3_STREAM_CROSSFADE_MS"] = str(args.stream_crossfade_ms)
   655	    os.environ["VEENA3_STREAM_ADAPTIVE_DECODE"] = "1" if args.adaptive_decode else "0"
   656	    os.environ["VEENA3_STREAM_DECODE_INTERVAL_FIRST"] = str(args.adaptive_decode_first_interval)
   657	    os.environ["VEENA3_STREAM_DECODE_INTERVAL_BUSY"] = str(args.adaptive_decode_busy_interval)
   658	    os.environ["VEENA3_STREAM_DECODE_BUSY_PENDING"] = str(args.adaptive_decode_busy_pending)
   659	    os.environ["VEENA3_STREAM_WINDOWED_DECODE"] = "0" if args.disable_windowed_decode else "1"
   660	    os.environ["VEENA3_BICODEC_BATCHING"] = "0" if args.disable_bicodec_batching else "1"
   661	    os.environ["VEENA3_BICODEC_BATCH_MAX"] = str(args.bicodec_batch_max)
   662	    os.environ["VEENA3_BICODEC_BATCH_TIMEOUT_MS"] = str(args.bicodec_batch_timeout_ms)
   663
   664	    enable_chunked_prefill = args.enable_chunked_prefill
   665	    if args.disable_chunked_prefill:
   666	        enable_chunked_prefill = False
   667
   668	    init_runtime(
   669	        model_path=model_path,
   670	        gpu_mem=args.gpu_memory,
   671	        stream_output_kind=args.stream_output_kind,
   672	        num_engines=args.num_engines,
   673	        max_num_batched_tokens=args.max_num_batched_tokens,
   674	        max_num_seqs=args.max_num_seqs,
   675	        enable_chunked_prefill=enable_chunked_prefill,
   676	        enable_prefix_caching=(False if args.disable_prefix_caching else None),
   677	        disable_log_stats=(True if args.disable_engine_stats_logs else None),
   678	        enforce_eager=(True if args.enforce_eager else None),
   679	    )
   680	    asyncio.run(main_async(args))
   681
   682
   683	if __name__ == "__main__":
   684	    main()


**Evaluating performance improvements**


The deterministic ladder finished successfully (`/tmp/stream_final_ladder_seeded.json`). I’m now generating a final delta summary versus the old baseline so we can decide the next optimization target from measured bottlenecks.


**Running script for delta**


python3 - <<'PY'
import json
base=json.load(open('/tmp/stream_baseline_levels_detailed.json'))['levels']
new=json.load(open('/tmp/stream_final_ladder_seeded.json'))['levels']
b={r['concurrency']:r for r in base}
n={r['concurrency']:r for r in new}
for c in [32,64,96]:
    rb=b[c]; rn=n[c]
    def pct(new,old):
        return (new-old)/old*100 if old else 0
    print(f"c={c} thr {rb['throughput_rps']:.2f}->{rn['throughput_rps']:.2f} ({pct(rn['throughput_rps'],rb['throughput_rps']):+.1f}%) | lat {rb['latency_ms']['avg']:.0f}->{rn['latency_ms']['avg']:.0f} ({pct(rn['latency_ms']['avg'],rb['latency_ms']['avg']):+.1f}%) | ttfb {rb['ttfb_ms']['avg']:.0f}->{rn['ttfb_ms']['avg']:.0f} ({pct(rn['ttfb_ms']['avg'],rb['ttfb_ms']['avg']):+.1f}%)")
PY


Output:
Chunk ID: cd8071
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 62
Output:
c=32 thr 12.56->13.36 (+6.4%) | lat 2546->1957 (-23.1%) | ttfb 1047->680 (-35.1%)
c=64 thr 16.77->21.67 (+29.2%) | lat 3812->2666 (-30.1%) | ttfb 1378->819 (-40.6%)
c=96 thr 18.92->24.07 (+27.2%) | lat 5069->3378 (-33.4%) | ttfb 1979->984 (-50.3%)


**Planning metrics analysis**


python3 - <<'PY'
import json
levels=json.load(open('/tmp/stream_final_ladder_seeded.json'))['levels']
for r in levels:
    c=r['concurrency'];t=r['timing']
    print(f"c={c} qwait={t['batch_queue_wait_ms_avg']['avg']:.1f}ms qdepth={t['batch_queue_depth_avg']['avg']:.1f} decode_int_p50={t['bicodec_decode_interval_ms_p50']['avg']:.1f}ms bicodec_calls={t['bicodec_decode_calls']['avg']:.2f} util={r['gpu']['gpu_util_pct']['avg']:.1f}%")
PY


Output:
Chunk ID: 1d2873
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 63
Output:
c=32 qwait=83.3ms qdepth=6.7 decode_int_p50=270.6ms bicodec_calls=4.50 util=55.2%
c=64 qwait=170.9ms qdepth=16.2 decode_int_p50=518.9ms bicodec_calls=4.02 util=61.2%
c=96 qwait=267.8ms qdepth=29.2 decode_int_p50=673.7ms bicodec_calls=4.15 util=78.7%


**Analyzing engine teardown and timeline shares**


python3 - <<'PY'
import json
levels=json.load(open('/tmp/stream_final_ladder_seeded.json'))['levels']
for r in levels:
    c=r['concurrency'];t=r['timing']
    bw=t['batch_queue_wait_ms_avg']['avg']
    bc=t['batch_compute_ms_avg']['avg']
    print(f"c={c} batch_queue_wait_avg={bw:.1f} batch_compute_avg={bc:.1f}")
PY


Output:
Chunk ID: 3a6c0a
Wall time: 0.0503 seconds
Process exited with code 0
Original token count: 42
Output:
c=32 batch_queue_wait_avg=83.3 batch_compute_avg=175.4
c=64 batch_queue_wait_avg=170.9 batch_compute_avg=99.0
c=96 batch_queue_wait_avg=267.8 batch_compute_avg=81.1


**Planning high-concurrency tests**


I have a stable ladder up to `c=96`; I’m doing one quick extension to `c=128` (same seeded config) to see whether throughput is still scaling or already flattening.


**Running single benchmark**


python3 scripts/stress_test_streaming_runtime.py --levels 128 --text short --max-tokens 1024 --seed-base 2026 --timeout 300 --requests-multiplier 1 --min-requests 8 --gpu-memory 0.25 --stream-output-kind delta --stream-decode-interval 48 --stream-window-size 128 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --output /tmp/stream_c128_seeded.json


Output:
Chunk ID: 51ea8d
Wall time: 1.0011 seconds
Process running with session ID 25184
Original token count: 0
Output:


Summary: auto


**Planning polling updates**


Output:
Chunk ID: a71363
Wall time: 5.0014 seconds
Process running with session ID 25184
Original token count: 1361
Output:
Total output lines: 51

[2026-02-18 18:40:32] INFO tts_runtime.py:183: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.25
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-18 18:40:33 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-18 18:40:33 [model.py:1561] Using max model len 4096
INFO 02-18 18:40:33 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-18 18:40:33 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:34 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=…161 tokens truncated…etailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:36 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:40339 backend=nccl
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:36 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A


**Preparing incremental updates**


The `c=128` extension is running; once this completes we’ll know whether throughput still scales past 96 or if we’re at the practical saturation point with current scheduling.


Output:
Chunk ID: 148a02
Wall time: 5.0008 seconds
Process running with session ID 25184
Original token count: 420
Output:
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:37 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:37 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=97255) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=97255) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=97255)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=97255)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.35s/it]
(EngineCore_DP0 pid=97255)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.35s/it]
(EngineCore_DP0 pid=97255)
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:39 [default_loader.py:291] Loading weights took 1.43 seconds
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:40 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.967182 seconds
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:44 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:44 [backends.py:872] Dynamo bytecode transform time: 4.18 s


Output:
Chunk ID: 822373
Wall time: 5.0011 seconds
Process running with session ID 25184
Original token count: 782
Output:
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:47 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.634 s
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:47 [monitor.py:34] torch.compile takes 4.82 s in total
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:48 [gpu_worker.py:356] Available KV cache memory: 18.06 GiB
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:48 [kv_cache_utils.py:1307] GPU KV cache size: 1,577,712 tokens
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:48 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 385.18x
(EngineCore_DP0 pid=97255)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 33.09it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 34.38it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 35.29it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 35.53it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 36.52it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 37.18it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 37.31it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 35.60it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:00<00:00, 35.01it/s]
(EngineCore_DP0 pid=97255)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 29.90it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 34.02it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 35.39it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 35.98it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 36.47it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.62it/s]
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:50 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=97255) INFO 02-18 18:40:50 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.71 seconds
[2026-02-18 18:40:52] INFO tts_runtime.py:192: Initializing prompt builder...
[2026-02-18 18:40:52] INFO tts_runtime.py:199: Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


Output:
Chunk ID: 96191c
Wall time: 5.0014 seconds
Process running with session ID 25184
Original token count: 127
Output:
[2026-02-18 18:40:59] INFO tts_runtime.py:209: Initializing TTS pipeline...
[2026-02-18 18:40:59] INFO tts_runtime.py:220: Initializing streaming pipeline...
[2026-02-18 18:40:59] INFO long_text_processor.py:112: LongTextProcessor initialized
[2026-02-18 18:40:59] INFO tts_runtime.py:277: ✅ TTS runtime initialized in 32449ms
[2026-02-18 18:40:59] INFO tts_runtime.py:278:    Model version: LLM
[2026-02-18 18:40:59] INFO tts_runtime.py:289: Speaker globals cache startup precompute skipped (disabled)


**Running benchmark tests**


Startup completed for the `c=128` run; now it’s in the actual load phase. I’ll report the saturation result as soon as it writes the level summary.


Output:
Chunk ID: 85eec0
Wall time: 0.7972 seconds
Process exited with code 0
Original token count: 672
Output:
✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cuda
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
   ℹ  Micro-batching enabled (max_batch=16, timeout_ms=6.0)
🚀 SparkTTSPipeline initialized
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)

=== streaming concurrency=128 requests=128 ===
success=128/128 (100%)
wall=5.42s throughput=23.62 req/s
latency_ms avg=4636 p50=4910 p95=5416 max=5416
ttfb_ms avg=1369 p50=1340 p95=1644 max=1645
chunks_per_req avg=5.1 p50=5.0 p95=6.0 max=6.0
inter_chunk_ms avg=787.8 p50=728.1 p95=2093.0 max=2449.4
tail_emit_speed_x avg=1.23 p50=1.11 p95=1.71 max=2.02
timeline_to_first_batch_ms: avg=66.59 p50=68.08 p95=78.39 max=79.72
timeline_to_first_chunk_emitted_ms: avg=1342.94 p50=1315.52 p95=1597.18 max=1598.90
timeline_total_ms: avg=4561.95 p50=4906.20 p95=5370.11 max=5375.10
llm_time_in_queue_ms: avg=0.16 p50=0.13 p95=0.36 max=0.40
llm_scheduler_ms: avg=46.11 p50=41.69 p95=59.74 max=59.74
llm_first_token_ms: avg=62.81 p50=64.50 p95=75.43 max=77.54
llm_request_lifecycle_ms: avg=2997.47 p50=3026.24 p95=3207.62 max=3321.93
llm_generation_wall_ms: avg=4547.60 p50=4873.97 p95=5370.08 max=5375.05
llm_time_per_token_ms: avg=19.84 p50=20.71 p95=22.88 max=23.92
llm_batch_wall_ms_total: avg=3941.53 p50=3908.73 p95=4922.00 max=5148.52
llm_decode_wall_ms_total: avg=3874.97 p50=3842.41 p95=4842.32 max=5081.71
tokens_per_batch_p50: avg=1.00 p50=1.00 p95=1.00 max=1.00
bicodec_decode_calls: avg=4.09 p50=4.00 p95=5.00 max=5.00
bicodec_decode_wall_ms_total: avg=3141.62 p50=3363.32 p95=4506.36 max=4544.95
bicodec_decode_interval_ms_p50: avg=960.90 p50=829.17 p95=1534.04 max=1594.23
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
batch_avg_size: avg=10.35 p50=9.46 p95=12.96 max=13.45
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_queue_wait_ms_avg: avg=653.29 p50=695.75 p95=788.33 max=788.33
batch_compute_ms_avg: avg=97.50 p50=86.03 p95=136.48 max=136.48
batch_queue_depth_avg: avg=71.33 p50=71.70 p95=74.73 max=75.08
latency_over_timeline_ms: avg=74.45 p50=42.48 p95=227.67 max=384.92
timeline_share: llm=99.7% bicodec=68.9% other=0.0%
gpu: util_avg=71% util_max=97% mem_avg=25809MB mem_max=26929MB

results saved: /tmp/stream_c128_seeded.json
ERROR 02-18 18:41:06 [core_client.py:605] Engine core proc EngineCore_DP0 died unexpectedly, shutting down client.


**Analyzing concurrency saturation around 96**


python3 - <<'PY'
import json
rows=[]
for p in ['/tmp/stream_final_ladder_seeded.json','/tmp/stream_c128_seeded.json']:
    d=json.load(open(p))
    for r in d['levels']:
        t=r['timing']
        rows.append((r['concurrency'],r['throughput_rps'],r['latency_ms']['avg'],r['ttfb_ms']['avg'],t['llm_time_per_token_ms']['avg'],t['bicodec_decode_wall_ms_total']['avg'],t['batch_queue_wait_ms_avg']['avg'],t['batch_queue_depth_avg']['avg'],r['gpu']['gpu_util_pct']['avg']))
rows=sorted(rows,key=lambda x:x[0])
for c,thr,lat,ttfb,llm,bic,qw,qd,util in rows:
    print(f"c={c} thr={thr:.2f} lat={lat:.0f} ttfb={ttfb:.0f} llm_tok_ms={llm:.2f} bicodec={bic:.0f} qwait={qw:.0f} qdepth={qd:.1f} util={util:.1f}")
PY


Output:
Chunk ID: 760931
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 95
Output:
c=32 thr=13.36 lat=1957 ttfb=680 llm_tok_ms=8.25 bicodec=820 qwait=83 qdepth=6.7 util=55.2
c=64 thr=21.67 lat=2666 ttfb=819 llm_tok_ms=11.45 bicodec=1099 qwait=171 qdepth=16.2 util=61.2
c=96 thr=24.07 lat=3378 ttfb=984 llm_tok_ms=14.51 bicodec=1904 qwait=268 qdepth=29.2 util=78.7
c=128 thr=23.62 lat=4636 ttfb=1369 llm_tok_ms=19.84 bicodec=3142 qwait=653 qdepth=71.3 util=70.7


**Summarizing final optimizations**


Implemented and benchmarked.

**Code changes**
- Added decode-queue observability in `veena3modal/core/bicodec_decoder.py:119` and `veena3modal/core/bicodec_decoder.py:371`:
  - `batch_queue_wait_ms_avg/min/max`
  - `batch_compute_ms_avg/min/max`
  - `batch_queue_depth_avg/max`
  - `batch_pending_now`
  - `get_pending_requests()` at `veena3modal/core/bicodec_decoder.py:441`
- Added adaptive decode scheduling hooks + metrics in `veena3modal/core/streaming_pipeline.py:65`, `veena3modal/core/streaming_pipeline.py:195`, `veena3modal/core/streaming_pipeline.py:526`, `veena3modal/core/streaming_pipeline.py:660`, `veena3modal/core/streaming_pipeline.py:1119`, `veena3modal/core/streaming_pipeline.py:1241`
  - `decode_interval_applied_tokens_*`
  - `decode_pending_requests_*`
  - env knobs for first/busy interval and pending threshold
- Extended benchmark harness in `scripts/stress_test_streaming_runtime.py:253`, `scripts/stress_test_streaming_runtime.py:404`, `scripts/stress_test_streaming_runtime.py:615`, `scripts/stress_test_streaming_runtime.py:650`
  - new timing fields above printed/saved
  - adaptive flags
  - deterministic seeding via `--seed-base`

**Deterministic stress ladder (current optimized path, seeded)**
- Output: `/tmp/stream_final_ladder_seeded.json` + `/tmp/stream_c128_seeded.json`
- Settings: decode interval 48, window 128, BiCodec batch max 16, timeout 6ms, adaptive off

- `c=32`: `13.36 req/s`, latency `1957 ms`, TTFB `680 ms`, queue wait `83 ms`
- `c=64`: `21.67 req/s`, latency `2666 ms`, TTFB `819 ms`, queue wait `171 ms`
- `c=96`: `24.07 req/s`, latency `3378 ms`, TTFB `984 ms`, queue wait `268 ms`
- `c=128`: `23.62 req/s`, latency `4636 ms`, TTFB `1369 ms`, queue wait `653 ms`

**What this says**
- Peak practical concurrency is around **96** with current architecture.
- At 128, throughput drops while latency/TTFB spike: decode queueing becomes dominant (`qdepth ~71`, `qwait ~653ms`).
- Compared with old baseline (`/tmp/stream_baseline_levels_detailed.json`) at `c=96`:
  - throughput `18.92 -> 24.07` (**+27.2%**)
  - latency `5069 -> 3378 ms` (**-33.4%**)
  - TTFB `1979 -> 984 ms` (**-50.3%**)

**Answer to “can we do better?”**
Yes. Next big win is not vLLM queue tuning; it is decode-side architecture:
1. Replace single decode worker with multi-stream/multi-worker decode scheduling (or dedicated decode process).
2. Add strict admission/SLO tiers so queue depth never reaches the 60–70 range.
3. Keep interval 48 for production latency; use 64 only for aggressive throughput tier (it can peak higher but hurts TTFB/stability).

If you want, I’ll implement step 1 next (multi-worker BiCodec decode scheduler) and rerun the same ladder.


sure.


**Planning multi-worker decode scheduler**


I’ll implement a multi-worker BiCodec micro-batch scheduler (shared queue, configurable worker count) and expose a benchmark flag/env for worker count, then run a seeded A/B at `c=96` and `c=128` to verify whether queue-wait drops and throughput improves.


rg -n "_batch_workers|_ensure_batch_worker|_batch_worker|BICODEC_BATCH" veena3modal/core/bicodec_decoder.py scripts/stress_test_streaming_runtime.py veena3modal/services/tts_runtime.py


sed -n '90,210p' veena3modal/core/bicodec_decoder.py


sed -n '340,470p' veena3modal/core/bicodec_decoder.py


sed -n '620,720p' scripts/stress_test_streaming_runtime.py


Output:
Chunk ID: 63bb06
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 464
Output:
veena3modal/services/tts_runtime.py:200:        batch_raw = os.environ.get("VEENA3_BICODEC_BATCHING", "true").strip().lower()
scripts/stress_test_streaming_runtime.py:660:    os.environ["VEENA3_BICODEC_BATCHING"] = "0" if args.disable_bicodec_batching else "1"
scripts/stress_test_streaming_runtime.py:661:    os.environ["VEENA3_BICODEC_BATCH_MAX"] = str(args.bicodec_batch_max)
scripts/stress_test_streaming_runtime.py:662:    os.environ["VEENA3_BICODEC_BATCH_TIMEOUT_MS"] = str(args.bicodec_batch_timeout_ms)
veena3modal/core/bicodec_decoder.py:111:        if _env_flag("VEENA3_BICODEC_BATCHING_FORCE_ON", False):
veena3modal/core/bicodec_decoder.py:113:        if _env_flag("VEENA3_BICODEC_BATCHING_FORCE_OFF", False):
veena3modal/core/bicodec_decoder.py:116:        self.batch_max_size = _env_int("VEENA3_BICODEC_BATCH_MAX", 16, minimum=1)
veena3modal/core/bicodec_decoder.py:117:        self.batch_timeout_s = _env_float("VEENA3_BICODEC_BATCH_TIMEOUT_MS", 6.0, minimum=0.1) / 1000.0
veena3modal/core/bicodec_decoder.py:122:        self._batch_workers: Dict[int, asyncio.Task] = {}
veena3modal/core/bicodec_decoder.py:357:    def _ensure_batch_worker(self, loop: asyncio.AbstractEventLoop) -> asyncio.Queue:
veena3modal/core/bicodec_decoder.py:361:            worker = self._batch_workers.get(key)
veena3modal/core/bicodec_decoder.py:366:            worker = loop.create_task(self._batch_worker(loop, queue), name=f"bicodec-batch-worker-{key}")
veena3modal/core/bicodec_decoder.py:368:            self._batch_workers[key] = worker
veena3modal/core/bicodec_decoder.py:371:    async def _batch_worker(self, loop: asyncio.AbstractEventLoop, queue: asyncio.Queue) -> None:
veena3modal/core/bicodec_decoder.py:439:                self._batch_workers.pop(key, None)
veena3modal/core/bicodec_decoder.py:610:            queue = self._ensure_batch_worker(loop)