Output:
Chunk ID: 0ceb3d
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 3373
Output:
   620	                                    previous_chunk_tail,
   621	                                    new_audio_bytes,
   622	                                    sample_rate_hz=16000,
   623	                                    crossfade_ms=CROSSFADE_MS,
   624	                                )
   625
   626	                                samples_emitted_in_this_chunk = len(to_emit) // 2
   627	                                total_samples_emitted_to_user += samples_emitted_in_this_chunk
   628
   629	                                if to_emit:
   630	                                    total_audio_chunks += 1
   631	                                    # Yield tuple: (audio_bytes, global_ids)
   632	                                    yield (to_emit, captured_globals)
   633
   634	        # Final chunk processing
   635	        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   636	            audio_bytes = self.snac_decoder.decode_streaming(
   637	                semantic_ids=semantic_buffer,
   638	                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
   639	                use_sliding_window=False,
   640	                trim_warmup=False
   641	            )
   642	            if audio_bytes:
   643	                total_samples_decoded = len(audio_bytes) // 2
   644
   645	                if total_samples_decoded > total_samples_emitted_to_user:
   646	                    new_bytes_start = total_samples_emitted_to_user * 2
   647	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   648
   649	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   650	                        previous_chunk_tail,
   651	                        new_audio_bytes,
   652	                        sample_rate_hz=16000,
   653	                        crossfade_ms=CROSSFADE_MS,
   654	                    )
   655
   656	                    samples_emitted_in_final = len(to_emit) // 2
   657	                    total_samples_emitted_to_user += samples_emitted_in_final
   658
   659	                    if to_emit:
   660	                        total_audio_chunks += 1
   661	                        yield (to_emit, captured_globals)
   662
   663	        # Flush remaining tail
   664	        if previous_chunk_tail:
   665	            total_audio_chunks += 1
   666	            tail_samples = len(previous_chunk_tail) // 2
   667	            total_samples_emitted_to_user += tail_samples
   668	            yield (previous_chunk_tail, captured_globals)
   669
   670	        t_end = time.time()
   671	        audio_duration_s = total_samples_emitted_to_user / 16000
   672	        logger.debug(
   673	            "first-chunk complete audio=%.2fs chunks=%d captured_globals=%d",
   674	            audio_duration_s,
   675	            total_audio_chunks,
   676	            len(captured_globals),
   677	        )
   678
   679	    async def generate_speech_stream_indic_continuation(
   680	        self,
   681	        speaker: str,
   682	        text: str,
   683	        global_ids: List[int],
   684	        temperature: float = DEFAULT_TEMPERATURE,
   685	        top_k: int = DEFAULT_TOP_K,
   686	        top_p: float = DEFAULT_TOP_P,
   687	        max_tokens: int = DEFAULT_MAX_TOKENS,
   688	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   689	        seed: Optional[int] = None,
   690	    ) -> AsyncGenerator[bytes, None]:
   691	        """
   692	        Generate speech for CONTINUATION chunks using pre-captured global tokens.
   693
   694	        This is the key method for voice consistency in chunked generation:
   695	        - Uses global_ids from the first chunk to maintain identical voice
   696	        - Model generates only semantic tokens (skips global token generation)
   697	        - Ensures no voice drift across chunks
   698
   699	        CRITICAL for production:
   700	        - global_ids is request-scoped, passed explicitly (no shared state)
   701	        - Thread-safe: each request has its own state
   702	        - No global caching that could cause cross-request contamination
   703
   704	        Args:
   705	            speaker: Speaker name (must match first chunk)
   706	            text: Continuation text chunk to synthesize
   707	            global_ids: 32 global token IDs captured from first chunk
   708	            temperature: Sampling temperature
   709	            top_k: Top-k sampling
   710	            top_p: Nucleus sampling
   711	            max_tokens: Max tokens to generate
   712	            repetition_penalty: Prevent repetition
   713	            seed: Random seed (use same as first chunk for consistency)
   714
   715	        Yields:
   716	            Audio bytes (int16 PCM, 16kHz mono)
   717
   718	        Raises:
   719	            ValueError: If global_ids doesn't contain exactly 32 tokens
   720
   721	        Thread Safety:
   722	            Fully thread-safe. All state is request-scoped and passed explicitly.
   723	        """
   724	        import time
   725	        t_start = time.time()
   726
   727	        # Validate global tokens
   728	        EXPECTED_GLOBAL_COUNT = 32
   729	        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
   730	            raise ValueError(
   731	                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
   732	                f"global_ids must be captured from first chunk generation."
   733	            )
   734
   735	        # Build prompt WITH pre-filled global tokens
   736	        # This tells the model to skip global generation and go straight to semantic tokens
   737	        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)
   738
   739	        # Configure sampling
   740	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
   741	        sampling_params = SamplingParams(
   742	            temperature=temperature,
   743	            top_k=top_k,
   744	            top_p=top_p,
   745	            max_tokens=max_tokens,
   746	            min_tokens=DEFAULT_MIN_TOKENS,
   747	            repetition_penalty=repetition_penalty,
   748	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
   749	            skip_special_tokens=False,
   750	            seed=seed,
   751	            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
   752	        )
   753
   754	        # BiCodec token buffers
   755	        semantic_buffer = []
   756	        global_buffer_unused = []  # Not used in continuation, but needed for parser API
   757	        processed_token_count = 0  # Used for cumulative mode; ignored for delta mode.
   758	        total_audio_chunks = 0
   759
   760	        # BiCodec streaming configuration
   761	        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
   762	        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
   763	        CROSSFADE_MS = 50
   764
   765	        # Generate unique request ID
   766	        import uuid
   767	        request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
   768
   769	        # Reuse pre-warmed token parser (singleton, created once in __init__)
   770	        token_parser = self.token_parser
   771
   772	        results_generator = self.model.engine.generate(
   773	            prompt=prompt,
   774	            sampling_params=sampling_params,
   775	            request_id=request_id,
   776	        )
   777
   778	        # Track state
   779	        last_decode_count = 0
   780	        total_samples_emitted_to_user = 0
   781	        previous_chunk_tail: Optional[bytes] = None
   782
   783	        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
   784	        async for request_output in results_generator:
   785	            generated_ids = request_output.outputs[0].token_ids
   786
   787	            # In DELTA mode token_ids are already incremental.
   788	            if _STREAM_USE_DELTA_OUTPUT:
   789	                new_token_ids = generated_ids
   790	            else:
   791	                new_token_ids = generated_ids[processed_token_count:]
   792	                processed_token_count = len(generated_ids)
   793	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)
   794
   795	            semantic_count = len(semantic_buffer)
   796	            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   797	                if semantic_count - last_decode_count >= DECODE_INTERVAL:
   798	                    last_decode_count = semantic_count
   799
   800	                    # Use pre-captured global tokens for decoding
   801	                    if self.snac_decoder.enable_batching:
   802	                        audio_bytes = await self.snac_decoder.decode_single_async(
   803	                            semantic_ids=semantic_buffer,
   804	                            global_ids=global_ids,  # Use captured globals
   805	                            trim_warmup=False,
   806	                            use_sliding_window=False
   807	                        )
   808	                    else:
   809	                        audio_bytes = self.snac_decoder.decode_streaming(
   810	                            semantic_ids=semantic_buffer,
   811	                            global_ids=global_ids,  # Use captured globals
   812	                            use_sliding_window=False,
   813	                            trim_warmup=False
   814	                        )
   815
   816	                    if audio_bytes:
   817	                        total_samples_decoded = len(audio_bytes) // 2
   818
   819	                        if total_samples_decoded > total_samples_emitted_to_user:
   820	                            new_bytes_start = total_samples_emitted_to_user * 2
   821	                            new_audio_bytes = audio_bytes[new_bytes_start:]
   822
   823	                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
   824	                                previous_chunk_tail,
   825	                                new_audio_bytes,
   826	                                sample_rate_hz=16000,
   827	                                crossfade_ms=CROSSFADE_MS,
   828	                            )
   829
   830	                            samples_emitted_in_this_chunk = len(to_emit) // 2
   831	                            total_samples_emitted_to_user += samples_emitted_in_this_chunk
   832
   833	                            if to_emit:
   834	                                total_audio_chunks += 1
   835	                                yield to_emit
   836
   837	        # Final chunk processing
   838	        if len(semantic_buffer) > last_decode_count:
   839	            audio_bytes = self.snac_decoder.decode_streaming(
   840	                semantic_ids=semantic_buffer,
   841	                global_ids=global_ids,  # Use captured globals
   842	                use_sliding_window=False,
   843	                trim_warmup=False
   844	            )
   845	            if audio_bytes:
   846	                total_samples_decoded = len(audio_bytes) // 2
   847
   848	                if total_samples_decoded > total_samples_emitted_to_user:
   849	                    new_bytes_start = total_samples_emitted_to_user * 2
   850	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   851
   852	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   853	                        previous_chunk_tail,
   854	                        new_audio_bytes,
   855	                        sample_rate_hz=16000,
   856	                        crossfade_ms=CROSSFADE_MS,
   857	                    )
   858
   859	                    samples_emitted_in_final = len(to_emit) // 2
   860	                    total_samples_emitted_to_user += samples_emitted_in_final
   861
   862	                    if to_emit:
   863	                        total_audio_chunks += 1
   864	                        yield to_emit
   865
   866	        # Flush remaining tail
   867	        if previous_chunk_tail:
   868	            total_audio_chunks += 1
   869	            tail_samples = len(previous_chunk_tail) // 2
   870	            total_samples_emitted_to_user += tail_samples
   871	            yield previous_chunk_tail
   872
   873	        t_end = time.time()
   874	        audio_duration_s = total_samples_emitted_to_user / 16000
   875	        logger.debug(
   876	            "continuation complete audio=%.2fs chunks=%d",
   877	            audio_duration_s,
   878	            total_audio_chunks,
   879	        )


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Planning incremental patching**


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py
@@
-import re
-from typing import AsyncGenerator, Optional, List
+import re
+from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union
@@
 _STREAM_OUTPUT_KIND_RAW = os.environ.get("VEENA3_STREAM_OUTPUT_KIND", "delta").strip().lower()
 _STREAM_USE_DELTA_OUTPUT = _STREAM_OUTPUT_KIND_RAW in {"delta", "d", "1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int, minimum: int = 0) -> int:
+    raw = os.environ.get(name, "")
+    if not raw:
+        return default
+    try:
+        value = int(raw)
+    except ValueError:
+        return default
+    return max(minimum, value)
+
+
+_STREAM_EXPECTED_GLOBAL_COUNT = 32
+_STREAM_MIN_SEMANTIC_FIRST_CHUNK = _env_int("VEENA3_STREAM_MIN_SEMANTIC_FIRST", 10, minimum=1)
+_STREAM_DECODE_INTERVAL = _env_int("VEENA3_STREAM_DECODE_INTERVAL", 48, minimum=1)
+_STREAM_CROSSFADE_MS = _env_int("VEENA3_STREAM_CROSSFADE_MS", 50, minimum=0)
+_STREAM_WINDOW_SIZE = _env_int("VEENA3_STREAM_WINDOW_SIZE", 128, minimum=16)
+_STREAM_WINDOWED_DECODE = os.environ.get("VEENA3_STREAM_WINDOWED_DECODE", "true").strip().lower() in {
+    "1",
+    "true",
+    "yes",
+    "on",
+}
@@
 class Veena3SlidingWindowPipeline:
@@
         self.token_parser = BiCodecTokenParser(tokenizer) if tokenizer else None

         print(f"🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)")
+
+    @staticmethod
+    def _p50(values: List[float]) -> float:
+        if not values:
+            return 0.0
+        ordered = sorted(values)
+        return ordered[(len(ordered) - 1) // 2]
+
+    @staticmethod
+    def _to_number(value: Any) -> Optional[float]:
+        if value is None:
+            return None
+        if isinstance(value, (int, float)):
+            return float(value)
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return None
+
+    def _extract_request_metrics(self, raw_metrics: Any) -> Dict[str, float]:
+        if raw_metrics is None:
+            return {}
+
+        numeric: Dict[str, float] = {}
+        items: List[Tuple[str, Any]] = []
+
+        if isinstance(raw_metrics, dict):
+            items.extend(raw_metrics.items())
+        else:
+            if hasattr(raw_metrics, "__dict__"):
+                try:
+                    items.extend(dict(raw_metrics.__dict__).items())
+                except Exception:
+                    pass
+            for name in dir(raw_metrics):
+                if name.startswith("_"):
+                    continue
+                try:
+                    value = getattr(raw_metrics, name)
+                except Exception:
+                    continue
+                if callable(value):
+                    continue
+                items.append((name, value))
+
+        seen: set[str] = set()
+        for key, value in items:
+            key_s = str(key)
+            if key_s in seen:
+                continue
+            seen.add(key_s)
+            parsed = self._to_number(value)
+            if parsed is not None:
+                numeric[key_s] = parsed
+        return numeric
+
+    def _apply_stats(self, perf: Dict[str, Any], prefix: str, values: List[float]) -> None:
+        if not values:
+            return
+        perf[f"{prefix}_total"] = float(sum(values))
+        perf[f"{prefix}_min"] = float(min(values))
+        perf[f"{prefix}_max"] = float(max(values))
+        perf[f"{prefix}_p50"] = float(self._p50(values))
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Summary: auto


**Reimplementing speech stream function**


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py
@@
     async def generate_speech_stream_indic(
         self,
         speaker: str,
         text: str,
         temperature: float = DEFAULT_TEMPERATURE,
         top_k: int = DEFAULT_TOP_K,  # Added for Spark TTS parity with non-streaming
         top_p: float = DEFAULT_TOP_P,
         max_tokens: int = DEFAULT_MAX_TOKENS,
         repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
         seed: Optional[int] = None,
-    ) -> AsyncGenerator[bytes, None]:
+        emit_progress: bool = False,
+    ) -> AsyncGenerator[Union[bytes, Tuple[bytes, Dict[str, Any]]], None]:
@@
-        import time
-        t_start = time.time()
-
-        # Build prompt using Indic prompt builder
-        prompt = self.prompt_builder.build_prefix(speaker, text)
-
-        # Configure sampling (matching non-streaming pipeline parameters)
-        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID (128258)
-        # which doesn't exist in the Spark vocab and would cause generation to run to max_tokens
-        sampling_params = SamplingParams(
-            temperature=temperature,
-            top_k=top_k,
-            top_p=top_p,
-            max_tokens=max_tokens,
-            min_tokens=DEFAULT_MIN_TOKENS,
-            repetition_penalty=repetition_penalty,
-            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
-            skip_special_tokens=False,  # Keep BiCodec tokens in output
-            seed=seed,
-            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
-        )
-
-        # BiCodec token buffers (separate for semantic and global)
-        semantic_buffer = []
-        global_buffer = []
-        processed_token_count = 0  # Used for cumulative mode; ignored for delta mode.
-        total_audio_chunks = 0
-
-        # BiCodec streaming configuration
-        EXPECTED_GLOBAL_COUNT = 32  # BiCodec uses 32 fixed global tokens (per paper)
-        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8, -120ms TTFB)
-        # OPTIMIZATION: Increased from 24 to 48 to reduce O(n) re-decode calls by 2x.
-        # At 50 TPS, 48 tokens = ~960ms audio per chunk. Cuts total decode calls roughly in half.
-        DECODE_INTERVAL = 48
-        CROSSFADE_MS = 50
-        # OPTIMIZATION: Windowed decode - only decode last WINDOW_SIZE tokens instead of ALL.
-        # BiCodec WaveGenerator has finite receptive field (~64 tokens based on kernel sizes).
-        # Window must be >= receptive_field + DECODE_INTERVAL to avoid boundary artifacts.
-        WINDOW_SIZE = 128  # 128 tokens covers receptive field + margin for crossfade
-
-        # Generate tokens with vLLM (streaming)
-        import uuid
-        request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
-
-        # Reuse pre-warmed token parser (singleton, created once in __init__)
-        token_parser = self.token_parser
-
-        t_gen_start = time.time()
-        results_generator = self.model.engine.generate(
-            prompt=prompt,
-            sampling_params=sampling_params,
-            request_id=request_id,
-        )
-
-        # Track timing
-        t_first_token = None
-        t_first_chunk_ready = None
-        t_first_chunk_decoded = None
-        iteration_count = 0
-        last_decode_count = 0
-
-        # Track audio samples EMITTED to the user (not just decoded)
-        # BiCodec produces ~320 samples per semantic token at 16kHz
-        # CRITICAL: This tracks samples that have been YIELDED, not held in tail
-        total_samples_emitted_to_user = 0
-        # Keep a tail to enable crossfading between consecutive emitted chunks
-        previous_chunk_tail: Optional[bytes] = None
-
-        # Stream tokens with BiCodec extraction
-        async for request_output in results_generator:
-            iteration_count += 1
-            t_iter = time.time()
-
-            # Track first token received
-            if t_first_token is None:
-                t_first_token = t_iter
-
-            # Extract generated token IDs
-            generated_ids = request_output.outputs[0].token_ids
-
-            # In DELTA mode token_ids are already incremental.
-            if _STREAM_USE_DELTA_OUTPUT:
-                new_token_ids = generated_ids
-            else:
-                new_token_ids = generated_ids[processed_token_count:]
-                processed_token_count = len(generated_ids)
-
-            # Parse new tokens and append to buffers (in-place modification)
-            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
-
-            # TRUE STREAMING with BiCodec two-phase generation:
-            # Phase 1: Generate 32 global tokens (pre-roll)
-            # Phase 2: Stream semantic tokens incrementally
-
-            # Check if we have the 32 global tokens (pre-roll complete)
-            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
-                # Pre-roll complete! Now we can stream semantic tokens
-                if t_first_chunk_ready is None and len(semantic_buffer) >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
-                    t_first_chunk_ready = time.time()
-
-                # Stream semantic tokens incrementally
-                # Only decode if we have enough NEW semantic tokens since last decode
-                semantic_count = len(semantic_buffer)
-                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
-                    # Check if we should decode (every DECODE_INTERVAL new semantic tokens)
-                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
-                        last_decode_count = semantic_count
-
-                        # OPTIMIZATION: Windowed decode instead of full re-decode
-                        # Only decode the last WINDOW_SIZE tokens (O(1) per chunk instead of O(n))
-                        # For first few chunks when total tokens < WINDOW_SIZE, decode all
-                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]
-
-                        if len(semantic_buffer) <= WINDOW_SIZE:
-                            # Early phase: decode all tokens (window not yet relevant)
-                            decode_semantic = semantic_buffer
-                        else:
-                            # Windowed: only decode last WINDOW_SIZE tokens
-                            decode_semantic = semantic_buffer[-WINDOW_SIZE:]
-
-                        t_decode_start = time.time()
-                        audio_bytes = await self.snac_decoder.decode_single_async(
-                            semantic_ids=decode_semantic,
-                            global_ids=window_global,
-                            trim_warmup=False,
-                            use_sliding_window=False,
-                        )
-                        t_decode_end = time.time()
-
-                        if audio_bytes:
-                            total_samples_decoded = len(audio_bytes) // 2
-
-                            if len(semantic_buffer) <= WINDOW_SIZE:
-                                # Early phase: extract new samples from full decode
-                                if total_samples_decoded > total_samples_emitted_to_user:
-                                    new_bytes_start = total_samples_emitted_to_user * 2
-                                    new_audio_bytes = audio_bytes[new_bytes_start:]
-                                else:
-                                    new_audio_bytes = b""
-                            else:
-                                # Windowed phase: take only the tail portion of the window decode
-                                # The last DECODE_INTERVAL tokens' worth of audio is the new content
-                                # Each token produces ~320 samples at 16kHz/50TPS
-                                new_samples_approx = DECODE_INTERVAL * 320
-                                if total_samples_decoded > new_samples_approx:
-                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
-                                else:
-                                    new_bytes_start = 0
-                                new_audio_bytes = audio_bytes[new_bytes_start:]
-
-                            if new_audio_bytes:
-                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
-                                    previous_chunk_tail,
-                                    new_audio_bytes,
-                                    sample_rate_hz=16000,
-                                    crossfade_ms=CROSSFADE_MS,
-                                )
-
-                                samples_emitted_in_this_chunk = len(to_emit) // 2
-                                total_samples_emitted_to_user += samples_emitted_in_this_chunk
-
-                                if to_emit:
-                                    total_audio_chunks += 1
-                                    if t_first_chunk_decoded is None:
-                                        t_first_chunk_decoded = t_decode_end
-
-                                    yield to_emit
-
-        # Final chunk: decode any remaining semantic tokens
-        # Use the same 32 global tokens (they get pooled)
-        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
-            audio_bytes = self.snac_decoder.decode_streaming(
-                semantic_ids=semantic_buffer,  # All semantic tokens
-                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],  # Use first 32 global tokens
-                use_sliding_window=False,  # Get all samples
-                trim_warmup=False
-            )
-            if audio_bytes:
-                # Only yield NEW samples
-                total_samples_decoded = len(audio_bytes) // 2
-
-                if total_samples_decoded > total_samples_emitted_to_user:
-                    new_bytes_start = total_samples_emitted_to_user * 2
-                    new_audio_bytes = audio_bytes[new_bytes_start:]
-
-                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
-                        previous_chunk_tail,
-                        new_audio_bytes,
-                        sample_rate_hz=16000,
-                        crossfade_ms=CROSSFADE_MS,
-                    )
-
-                    samples_emitted_in_final = len(to_emit) // 2
-                    total_samples_emitted_to_user += samples_emitted_in_final
-
-                    if to_emit:
-                        total_audio_chunks += 1
-                        yield to_emit
-
-        # Flush any remaining tail that was held back for crossfade
-        if previous_chunk_tail:
-            total_audio_chunks += 1
-            tail_samples = len(previous_chunk_tail) // 2
-            total_samples_emitted_to_user += tail_samples
-            yield previous_chunk_tail
-
-        # Log completion summary
-        t_end = time.time()
-        audio_duration_s = total_samples_emitted_to_user / 16000  # 16kHz sample rate
-        ttfb_ms = (t_first_chunk_decoded - t_start) * 1000 if t_first_chunk_decoded else 0
-        logger.debug(
-            "streaming complete audio=%.2fs ttfb=%.0fms chunks=%d",
-            audio_duration_s,
-            ttfb_ms,
-            total_audio_chunks,
-        )
+        import time
+        import uuid
+
+        t_request_start = time.perf_counter()
+        timeline_marks: List[Dict[str, float]] = [{"stage": "request_start", "t_ms": 0.0, "dt_ms": 0.0}]
+        last_mark_ms = 0.0
+        mark_stage_set: set[str] = {"request_start"}
+
+        def _mark(stage: str) -> float:
+            nonlocal last_mark_ms
+            now_ms = (time.perf_counter() - t_request_start) * 1000.0
+            dt_ms = now_ms - last_mark_ms
+            last_mark_ms = now_ms
+            timeline_marks.append({"stage": stage, "t_ms": float(now_ms), "dt_ms": float(dt_ms)})
+            mark_stage_set.add(stage)
+            return float(now_ms)
+
+        prompt_t0 = time.perf_counter()
+        prompt = self.prompt_builder.build_prefix(speaker, text)
+        prompt_build_ms = (time.perf_counter() - prompt_t0) * 1000.0
+        _mark("prompt_ready")
+
+        sampling_t0 = time.perf_counter()
+        sampling_params = SamplingParams(
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            max_tokens=max_tokens,
+            min_tokens=DEFAULT_MIN_TOKENS,
+            repetition_penalty=repetition_penalty,
+            stop=TRAINING_STOP_TOKEN_IDS,
+            skip_special_tokens=False,
+            seed=seed,
+            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
+        )
+        sampling_params_ms = (time.perf_counter() - sampling_t0) * 1000.0
+        _mark("sampling_ready")
+
+        request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
+        token_parser = self.token_parser
+        if token_parser is None:
+            tokenizer = getattr(self.model, "tokenizer", None) or getattr(self.model.engine, "tokenizer", None)
+            token_parser = BiCodecTokenParser(tokenizer) if tokenizer else None
+        if token_parser is None:
+            raise RuntimeError("BiCodecTokenParser unavailable for streaming")
+
+        results_generator = self.model.engine.generate(
+            prompt=prompt,
+            sampling_params=sampling_params,
+            request_id=request_id,
+        )
+
+        semantic_buffer: List[int] = []
+        global_buffer: List[int] = []
+        processed_token_count = 0
+        total_generated_tokens = 0
+        total_audio_chunks = 0
+        total_samples_emitted_to_user = 0
+        previous_chunk_tail: Optional[bytes] = None
+        last_decode_count = 0
+        last_decode_end_ts: Optional[float] = None
+        t_first_chunk_decode_done: Optional[float] = None
+        t_first_chunk_emitted: Optional[float] = None
+        final_request_metrics: Dict[str, float] = {}
+        prev_gpu_cumulative_s: Optional[float] = None
+
+        llm_batch_wall_ms: List[float] = []
+        llm_batch_gpu_ms: List[float] = []
+        llm_decode_wall_ms: List[float] = []
+        llm_decode_gpu_ms: List[float] = []
+        llm_parse_ms_samples: List[float] = []
+        tokens_per_batch: List[float] = []
+        bicodec_decode_wall_ms: List[float] = []
+        bicodec_decode_tokens: List[float] = []
+        bicodec_decode_intervals_ms: List[float] = []
+
+        t_llm_start = time.perf_counter()
+        t_last_batch = t_llm_start
+
+        async for request_output in results_generator:
+            now = time.perf_counter()
+            batch_wall = (now - t_last_batch) * 1000.0
+            t_last_batch = now
+
+            outputs = request_output.outputs or []
+            if not outputs:
+                continue
+
+            generated_ids = outputs[0].token_ids
+            if _STREAM_USE_DELTA_OUTPUT:
+                new_token_ids = generated_ids
+                processed_token_count += len(new_token_ids)
+            else:
+                new_token_ids = generated_ids[processed_token_count:]
+                processed_token_count = len(generated_ids)
+
+            new_token_count = len(new_token_ids)
+            if new_token_count <= 0:
+                continue
+
+            total_generated_tokens += new_token_count
+            llm_batch_wall_ms.append(float(batch_wall))
+            tokens_per_batch.append(float(new_token_count))
+
+            if "llm_first_batch" not in mark_stage_set:
+                _mark("llm_first_batch")
+            elif llm_batch_wall_ms:
+                llm_decode_wall_ms.append(float(batch_wall))
+
+            raw_req_metrics = self._extract_request_metrics(getattr(request_output, "metrics", None))
+            if raw_req_metrics:
+                final_request_metrics = raw_req_metrics
+                cumulative_gpu_s: Optional[float] = None
+                if "model_execute_time" in raw_req_metrics:
+                    cumulative_gpu_s = raw_req_metrics["model_execute_time"]
+                elif "model_forward_time" in raw_req_metrics:
+                    cumulative_gpu_s = raw_req_metrics["model_forward_time"]
+
+                if cumulative_gpu_s is not None:
+                    if prev_gpu_cumulative_s is None:
+                        gpu_delta_ms = cumulative_gpu_s * 1000.0
+                    elif cumulative_gpu_s >= prev_gpu_cumulative_s:
+                        gpu_delta_ms = (cumulative_gpu_s - prev_gpu_cumulative_s) * 1000.0
+                    else:
+                        gpu_delta_ms = cumulative_gpu_s * 1000.0
+                    prev_gpu_cumulative_s = cumulative_gpu_s
+                    if gpu_delta_ms >= 0.0:
+                        llm_batch_gpu_ms.append(float(gpu_delta_ms))
+                        if len(llm_batch_wall_ms) > 1:
+                            llm_decode_gpu_ms.append(float(gpu_delta_ms))
+
+            parse_t0 = time.perf_counter()
+            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
+            llm_parse_ms_samples.append((time.perf_counter() - parse_t0) * 1000.0)
+
+            if len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
+                semantic_count = len(semantic_buffer)
+                if semantic_count >= _STREAM_MIN_SEMANTIC_FIRST_CHUNK:
+                    if "first_chunk_ready" not in mark_stage_set:
+                        _mark("first_chunk_ready")
+
+                    if semantic_count - last_decode_count >= _STREAM_DECODE_INTERVAL:
+                        last_decode_count = semantic_count
+                        decode_global = global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT]
+
+                        if (
+                            _STREAM_WINDOWED_DECODE
+                            and semantic_count > _STREAM_WINDOW_SIZE
+                        ):
+                            decode_semantic = semantic_buffer[-_STREAM_WINDOW_SIZE:]
+                        else:
+                            decode_semantic = semantic_buffer
+
+                        decode_t0 = time.perf_counter()
+                        audio_bytes = await self.snac_decoder.decode_single_async(
+                            semantic_ids=decode_semantic,
+                            global_ids=decode_global,
+                            trim_warmup=False,
+                            use_sliding_window=False,
+                        )
+                        decode_t1 = time.perf_counter()
+
+                        decode_wall_ms = (decode_t1 - decode_t0) * 1000.0
+                        bicodec_decode_wall_ms.append(float(decode_wall_ms))
+                        bicodec_decode_tokens.append(float(len(decode_semantic)))
+                        if last_decode_end_ts is not None:
+                            bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
+                        last_decode_end_ts = decode_t1
+
+                        if audio_bytes:
+                            total_samples_decoded = len(audio_bytes) // 2
+                            if (
+                                _STREAM_WINDOWED_DECODE
+                                and semantic_count > _STREAM_WINDOW_SIZE
+                            ):
+                                new_samples_approx = _STREAM_DECODE_INTERVAL * 320
+                                if total_samples_decoded > new_samples_approx:
+                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
+                                else:
+                                    new_bytes_start = 0
+                                new_audio_bytes = audio_bytes[new_bytes_start:]
+                            else:
+                                if total_samples_decoded > total_samples_emitted_to_user:
+                                    new_bytes_start = total_samples_emitted_to_user * 2
+                                    new_audio_bytes = audio_bytes[new_bytes_start:]
+                                else:
+                                    new_audio_bytes = b""
+
+                            if new_audio_bytes:
+                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
+                                    previous_chunk_tail,
+                                    new_audio_bytes,
+                                    sample_rate_hz=16000,
+                                    crossfade_ms=_STREAM_CROSSFADE_MS,
+                                )
+                                if to_emit:
+                                    total_audio_chunks += 1
+                                    total_samples_emitted_to_user += len(to_emit) // 2
+                                    if t_first_chunk_decode_done is None:
+                                        t_first_chunk_decode_done = decode_t1
+                                        _mark("first_chunk_decoded")
+                                    if t_first_chunk_emitted is None:
+                                        t_first_chunk_emitted = time.perf_counter()
+                                        _mark("first_chunk_emitted")
+                                    if emit_progress:
+                                        yield (to_emit, {})
+                                    else:
+                                        yield to_emit
+
+        llm_generation_wall_ms = (time.perf_counter() - t_llm_start) * 1000.0
+        _mark("llm_done")
+
+        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
+            decode_semantic = semantic_buffer
+            decode_t0 = time.perf_counter()
+            audio_bytes = self.snac_decoder.decode_streaming(
+                semantic_ids=decode_semantic,
+                global_ids=global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT],
+                use_sliding_window=False,
+                trim_warmup=False,
+            )
+            decode_t1 = time.perf_counter()
+            decode_wall_ms = (decode_t1 - decode_t0) * 1000.0
+            bicodec_decode_wall_ms.append(float(decode_wall_ms))
+            bicodec_decode_tokens.append(float(len(decode_semantic)))
+            if last_decode_end_ts is not None:
+                bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
+            last_decode_end_ts = decode_t1
+
+            if audio_bytes:
+                total_samples_decoded = len(audio_bytes) // 2
+                if total_samples_decoded > total_samples_emitted_to_user:
+                    new_bytes_start = total_samples_emitted_to_user * 2
+                    new_audio_bytes = audio_bytes[new_bytes_start:]
+                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
+                        previous_chunk_tail,
+                        new_audio_bytes,
+                        sample_rate_hz=16000,
+                        crossfade_ms=_STREAM_CROSSFADE_MS,
+                    )
+                    if to_emit:
+                        total_audio_chunks += 1
+                        total_samples_emitted_to_user += len(to_emit) // 2
+                        if t_first_chunk_decode_done is None:
+                            t_first_chunk_decode_done = decode_t1
+                            _mark("first_chunk_decoded")
+                        if t_first_chunk_emitted is None:
+                            t_first_chunk_emitted = time.perf_counter()
+                            _mark("first_chunk_emitted")
+                        if emit_progress:
+                            yield (to_emit, {})
+                        else:
+                            yield to_emit
+
+        if previous_chunk_tail:
+            total_audio_chunks += 1
+            total_samples_emitted_to_user += len(previous_chunk_tail) // 2
+            if t_first_chunk_emitted is None:
+                t_first_chunk_emitted = time.perf_counter()
+                _mark("first_chunk_emitted")
+            if emit_progress:
+                yield (previous_chunk_tail, {})
+            else:
+                yield previous_chunk_tail
+
+        _mark("stream_done")
+        _mark("request_done")
+
+        perf: Dict[str, Any] = {
+            "prompt_build_ms": float(prompt_build_ms),
+            "sampling_params_ms": float(sampling_params_ms),
+            "llm_generation_wall_ms": float(llm_generation_wall_ms),
+            "llm_batch_count": int(len(llm_batch_wall_ms)),
+            "llm_token_total": int(total_generated_tokens),
+            "semantic_token_total": int(len(semantic_buffer)),
+            "global_token_total": int(len(global_buffer)),
+            "llm_parse_ms": float(sum(llm_parse_ms_samples)),
+            "bicodec_decode_calls": int(len(bicodec_decode_wall_ms)),
+            "llm_decode_calls": int(len(bicodec_decode_wall_ms)),
+            "chunks_sent": int(total_audio_chunks),
+            "audio_duration_seconds": float(total_samples_emitted_to_user / 16000.0),
+            "ttfb_ms": float((t_first_chunk_emitted - t_request_start) * 1000.0 if t_first_chunk_emitted else 0.0),
+            "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
+            "stream_decode_interval_tokens": int(_STREAM_DECODE_INTERVAL),
+            "stream_window_size_tokens": int(_STREAM_WINDOW_SIZE),
+            "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
+            "timeline_markers": list(timeline_marks),
+            "timeline_total_ms": float((time.perf_counter() - t_request_start) * 1000.0),
+        }
+
+        self._apply_stats(perf, "llm_batch_wall_ms", llm_batch_wall_ms)
+        self._apply_stats(perf, "llm_batch_gpu_ms", llm_batch_gpu_ms)
+        self._apply_stats(perf, "llm_decode_wall_ms", llm_decode_wall_ms)
+        self._apply_stats(perf, "llm_decode_gpu_ms", llm_decode_gpu_ms)
+        self._apply_stats(perf, "tokens_per_batch", tokens_per_batch)
+        self._apply_stats(perf, "llm_parse_step_ms", llm_parse_ms_samples)
+        self._apply_stats(perf, "bicodec_decode_wall_ms", bicodec_decode_wall_ms)
+        self._apply_stats(perf, "bicodec_decode_tokens", bicodec_decode_tokens)
+        self._apply_stats(perf, "bicodec_decode_interval_ms", bicodec_decode_intervals_ms)
+
+        perf["bicodec_decode_wall_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))
+        perf["bicodec_decode_gpu_ms"] = 0.0
+        perf["bicodec_decode_cpu_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))
+
+        if total_generated_tokens > 0:
+            perf["llm_time_per_token_ms"] = float(llm_generation_wall_ms) / float(total_generated_tokens)
+        if llm_batch_wall_ms:
+            perf["llm_time_per_batch_wall_ms"] = float(sum(llm_batch_wall_ms)) / float(len(llm_batch_wall_ms))
+        if llm_batch_gpu_ms:
+            perf["llm_time_per_batch_gpu_ms"] = float(sum(llm_batch_gpu_ms)) / float(len(llm_batch_gpu_ms))
+
+        for mark in timeline_marks:
+            stage = mark.get("stage")
+            if not stage:
+                continue
+            perf[f"timeline_{stage}_ms"] = float(mark.get("t_ms", 0.0))
+            perf[f"timeline_{stage}_delta_ms"] = float(mark.get("dt_ms", 0.0))
+
+        tl_first_batch = perf.get("timeline_llm_first_batch_ms")
+        tl_first_emit = perf.get("timeline_first_chunk_emitted_ms")
+        tl_done = perf.get("timeline_request_done_ms")
+        if isinstance(tl_first_batch, (int, float)):
+            perf["timeline_to_first_batch_ms"] = float(tl_first_batch)
+        if isinstance(tl_first_emit, (int, float)):
+            perf["timeline_to_first_chunk_emitted_ms"] = float(tl_first_emit)
+        if isinstance(tl_first_batch, (int, float)) and isinstance(tl_first_emit, (int, float)) and tl_first_emit >= tl_first_batch:
+            perf["timeline_first_batch_to_first_chunk_emitted_ms"] = float(tl_first_emit - tl_first_batch)
+        if isinstance(tl_first_emit, (int, float)) and isinstance(tl_done, (int, float)) and tl_done >= tl_first_emit:
+            perf["timeline_first_chunk_to_request_done_ms"] = float(tl_done - tl_first_emit)
+
+        sec_fields = {
+            "time_in_queue": "llm_time_in_queue_ms",
+            "scheduler_time": "llm_scheduler_ms",
+            "model_forward_time": "llm_model_forward_ms",
+            "model_execute_time": "llm_model_execute_ms",
+            "first_token_latency": "llm_first_token_ms",
+        }
+        for src, dst in sec_fields.items():
+            if src in final_request_metrics:
+                perf[dst] = float(final_request_metrics[src]) * 1000.0
+
+        def _delta_ms(start_s: Optional[float], end_s: Optional[float]) -> Optional[float]:
+            if start_s is None or end_s is None:
+                return None
+            if end_s < start_s:
+                return None
+            return (end_s - start_s) * 1000.0
+
+        queued_ts = final_request_metrics.get("queued_ts")
+        scheduled_ts = final_request_metrics.get("scheduled_ts")
+        first_token_ts = final_request_metrics.get("first_token_ts")
+        last_token_ts = final_request_metrics.get("last_token_ts")
+        q_to_sched = _delta_ms(queued_ts, scheduled_ts)
+        sched_to_first = _delta_ms(scheduled_ts, first_token_ts)
+        first_to_last = _delta_ms(first_token_ts, last_token_ts)
+        queued_to_last = _delta_ms(queued_ts, last_token_ts)
+        if q_to_sched is not None:
+            perf["llm_queued_to_scheduled_ms"] = q_to_sched
+            perf["llm_time_in_queue_ms"] = q_to_sched
+        if sched_to_first is not None:
+            perf["llm_scheduled_to_first_token_ms"] = sched_to_first
+            perf["llm_scheduler_ms"] = sched_to_first
+        if first_to_last is not None:
+            perf["llm_first_to_last_token_ms"] = first_to_last
+            perf["llm_model_execute_ms"] = first_to_last
+        if queued_to_last is not None:
+            perf["llm_queued_to_last_token_ms"] = queued_to_last
+            perf["llm_request_lifecycle_ms"] = queued_to_last
+
+        logger.debug(
+            "streaming complete audio=%.2fs ttfb=%.0fms chunks=%d llm_tokens=%d",
+            perf["audio_duration_seconds"],
+            perf["ttfb_ms"],
+            perf["chunks_sent"],
+            perf["llm_token_total"],
+        )
+
+        if emit_progress:
+            yield (b"", perf)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Extending continuation with progress and windowed decode**


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py
@@
     async def generate_speech_stream_indic_continuation(
         self,
         speaker: str,
         text: str,
         global_ids: List[int],
@@
         top_p: float = DEFAULT_TOP_P,
         max_tokens: int = DEFAULT_MAX_TOKENS,
         repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
         seed: Optional[int] = None,
-    ) -> AsyncGenerator[bytes, None]:
+        emit_progress: bool = False,
+    ) -> AsyncGenerator[Union[bytes, Tuple[bytes, Dict[str, Any]]], None]:
@@
-        # Validate global tokens
-        EXPECTED_GLOBAL_COUNT = 32
-        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
+        # Validate global tokens
+        if len(global_ids) != _STREAM_EXPECTED_GLOBAL_COUNT:
             raise ValueError(
-                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
+                f"Expected exactly {_STREAM_EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
                 f"global_ids must be captured from first chunk generation."
             )
@@
         semantic_buffer = []
         global_buffer_unused = []  # Not used in continuation, but needed for parser API
         processed_token_count = 0  # Used for cumulative mode; ignored for delta mode.
         total_audio_chunks = 0
+        total_generated_tokens = 0

         # BiCodec streaming configuration
-        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
-        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
-        CROSSFADE_MS = 50
+        MIN_SEMANTIC_FOR_FIRST_CHUNK = _STREAM_MIN_SEMANTIC_FIRST_CHUNK
+        DECODE_INTERVAL = _STREAM_DECODE_INTERVAL
+        CROSSFADE_MS = _STREAM_CROSSFADE_MS
+        WINDOW_SIZE = _STREAM_WINDOW_SIZE
@@
         results_generator = self.model.engine.generate(
             prompt=prompt,
             sampling_params=sampling_params,
             request_id=request_id,
         )
@@
         last_decode_count = 0
         total_samples_emitted_to_user = 0
         previous_chunk_tail: Optional[bytes] = None
+        t_first_chunk_emitted: Optional[float] = None
+        decode_calls = 0
+        decode_wall_ms_total = 0.0
@@
             if _STREAM_USE_DELTA_OUTPUT:
                 new_token_ids = generated_ids
+                processed_token_count += len(new_token_ids)
             else:
                 new_token_ids = generated_ids[processed_token_count:]
                 processed_token_count = len(generated_ids)
+            total_generated_tokens += len(new_token_ids)
             token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)

             semantic_count = len(semantic_buffer)
             if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                 if semantic_count - last_decode_count >= DECODE_INTERVAL:
                     last_decode_count = semantic_count
+                    if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
+                        decode_semantic = semantic_buffer[-WINDOW_SIZE:]
+                    else:
+                        decode_semantic = semantic_buffer

                     # Use pre-captured global tokens for decoding
+                    decode_t0 = time.perf_counter()
                     if self.snac_decoder.enable_batching:
                         audio_bytes = await self.snac_decoder.decode_single_async(
-                            semantic_ids=semantic_buffer,
+                            semantic_ids=decode_semantic,
                             global_ids=global_ids,  # Use captured globals
                             trim_warmup=False,
                             use_sliding_window=False
                         )
                     else:
                         audio_bytes = self.snac_decoder.decode_streaming(
-                            semantic_ids=semantic_buffer,
+                            semantic_ids=decode_semantic,
                             global_ids=global_ids,  # Use captured globals
                             use_sliding_window=False,
                             trim_warmup=False
                         )
+                    decode_calls += 1
+                    decode_wall_ms_total += (time.perf_counter() - decode_t0) * 1000.0

                     if audio_bytes:
                         total_samples_decoded = len(audio_bytes) // 2
-
-                        if total_samples_decoded > total_samples_emitted_to_user:
-                            new_bytes_start = total_samples_emitted_to_user * 2
-                            new_audio_bytes = audio_bytes[new_bytes_start:]
+                        if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
+                            new_samples_approx = DECODE_INTERVAL * 320
+                            if total_samples_decoded > new_samples_approx:
+                                new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
+                            else:
+                                new_bytes_start = 0
+                            new_audio_bytes = audio_bytes[new_bytes_start:]
+                        elif total_samples_decoded > total_samples_emitted_to_user:
+                            new_bytes_start = total_samples_emitted_to_user * 2
+                            new_audio_bytes = audio_bytes[new_bytes_start:]
+                        else:
+                            new_audio_bytes = b""

+                        if new_audio_bytes:
                             to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                 previous_chunk_tail,
                                 new_audio_bytes,
                                 sample_rate_hz=16000,
                                 crossfade_ms=CROSSFADE_MS,
                             )

-                            samples_emitted_in_this_chunk = len(to_emit) // 2
-                            total_samples_emitted_to_user += samples_emitted_in_this_chunk
-
                             if to_emit:
                                 total_audio_chunks += 1
-                                yield to_emit
+                                total_samples_emitted_to_user += len(to_emit) // 2
+                                if t_first_chunk_emitted is None:
+                                    t_first_chunk_emitted = time.perf_counter()
+                                if emit_progress:
+                                    yield (to_emit, {})
+                                else:
+                                    yield to_emit
@@
         if len(semantic_buffer) > last_decode_count:
+            decode_t0 = time.perf_counter()
             audio_bytes = self.snac_decoder.decode_streaming(
                 semantic_ids=semantic_buffer,
                 global_ids=global_ids,  # Use captured globals
                 use_sliding_window=False,
                 trim_warmup=False
             )
+            decode_calls += 1
+            decode_wall_ms_total += (time.perf_counter() - decode_t0) * 1000.0
             if audio_bytes:
                 total_samples_decoded = len(audio_bytes) // 2
@@
                     if to_emit:
                         total_audio_chunks += 1
-                        yield to_emit
+                        total_samples_emitted_to_user += len(to_emit) // 2
+                        if t_first_chunk_emitted is None:
+                            t_first_chunk_emitted = time.perf_counter()
+                        if emit_progress:
+                            yield (to_emit, {})
+                        else:
+                            yield to_emit
@@
         if previous_chunk_tail:
             total_audio_chunks += 1
             tail_samples = len(previous_chunk_tail) // 2
             total_samples_emitted_to_user += tail_samples
-            yield previous_chunk_tail
+            if t_first_chunk_emitted is None:
+                t_first_chunk_emitted = time.perf_counter()
+            if emit_progress:
+                yield (previous_chunk_tail, {})
+            else:
+                yield previous_chunk_tail
@@
         logger.debug(
             "continuation complete audio=%.2fs chunks=%d",
             audio_duration_s,
             total_audio_chunks,
         )
+
+        if emit_progress:
+            perf: Dict[str, Any] = {
+                "llm_token_total": int(total_generated_tokens),
+                "semantic_token_total": int(len(semantic_buffer)),
+                "global_token_total": int(len(global_ids)),
+                "bicodec_decode_calls": int(decode_calls),
+                "llm_decode_calls": int(decode_calls),
+                "bicodec_decode_wall_ms": float(decode_wall_ms_total),
+                "bicodec_decode_gpu_ms": 0.0,
+                "bicodec_decode_cpu_ms": float(decode_wall_ms_total),
+                "chunks_sent": int(total_audio_chunks),
+                "audio_duration_seconds": float(audio_duration_s),
+                "ttfb_ms": float((t_first_chunk_emitted - t_start) * 1000.0 if t_first_chunk_emitted else 0.0),
+                "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
+                "stream_decode_interval_tokens": int(DECODE_INTERVAL),
+                "stream_window_size_tokens": int(WINDOW_SIZE),
+                "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
+            }
+            yield (b"", perf)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Planning streaming patch integration**


cd /home/ubuntu/veenaModal && nl -ba veena3modal/services/tts_runtime.py | sed -n '760,980p'


Output:
Chunk ID: a122e8
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2079
Output:
   760	            runtime, long_processor, text, speaker, temperature, top_k, top_p,
   761	            max_tokens, repetition_penalty, seed, sample_rate
   762	        ):
   763	            # First chunk: prepend WAV header
   764	            if first_chunk_time is None:
   765	                first_chunk_time = time.time()
   766	                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
   767	                # Create streaming WAV header (size=0 for unknown length)
   768	                wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
   769	                audio_chunk = wav_header + audio_chunk
   770
   771	            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
   772	            metrics["chunks_sent"] += 1
   773	            metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
   774	            metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
   775
   776	            yield audio_chunk, metrics
   777	    else:
   778	        # Simple streaming (no text chunking)
   779	        # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
   780	        # use continuation mode to skip the ~110ms global token pre-roll.
   781	        # The model jumps straight to semantic token generation.
   782	        cached_globals = runtime.speaker_global_cache.get(speaker)
   783
   784	        if cached_globals:
   785	            # Fast path: use cached globals, skip global token generation entirely
   786	            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
   787	                speaker=speaker,
   788	                text=text,
   789	                global_ids=cached_globals,
   790	                temperature=temperature,
   791	                top_k=top_k,
   792	                top_p=top_p,
   793	                max_tokens=max_tokens,
   794	                repetition_penalty=repetition_penalty,
   795	                seed=seed,
   796	            )
   797	        else:
   798	            # Fallback: no cached globals, generate them inline (original path)
   799	            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
   800	                speaker=speaker,
   801	                text=text,
   802	                temperature=temperature,
   803	                top_k=top_k,
   804	                top_p=top_p,
   805	                max_tokens=max_tokens,
   806	                repetition_penalty=repetition_penalty,
   807	                seed=seed,
   808	            )
   809
   810	        async for audio_chunk in stream_gen:
   811	            # First chunk: prepend WAV header
   812	            if first_chunk_time is None:
   813	                first_chunk_time = time.time()
   814	                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
   815	                wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
   816	                audio_chunk = wav_header + audio_chunk
   817
   818	            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
   819	            metrics["chunks_sent"] += 1
   820	            metrics["total_bytes"] = total_pcm_bytes + 44
   821	            metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
   822
   823	            yield audio_chunk, metrics
   824
   825
   826	async def _stream_chunked_text(
   827	    runtime: TTSRuntime,
   828	    long_processor,
   829	    text: str,
   830	    speaker: str,
   831	    temperature: float,
   832	    top_k: int,
   833	    top_p: float,
   834	    max_tokens: int,
   835	    repetition_penalty: float,
   836	    seed: Optional[int],
   837	    sample_rate: int,
   838	) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
   839	    """
   840	    Internal helper: stream chunked text with voice consistency.
   841
   842	    Uses global token caching from first chunk to maintain voice across chunks.
   843	    """
   844	    from veena3modal.audio.crossfade import crossfade_bytes_int16
   845
   846	    # Chunk text using the long text processor's chunker
   847	    chunks = long_processor.chunk_text(text)
   848
   849	    if not chunks:
   850	        return
   851
   852	    captured_globals: Optional[List[int]] = None
   853	    previous_chunk_tail: Optional[bytes] = None
   854	    chunk_metrics = {"chunks_processed": 0}
   855
   856	    for i, chunk_text in enumerate(chunks):
   857	        chunk_metrics["chunks_processed"] = i + 1
   858
   859	        if i == 0:
   860	            # First chunk: capture global tokens for voice consistency
   861	            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
   862	                speaker=speaker,
   863	                text=chunk_text,
   864	                temperature=temperature,
   865	                top_k=top_k,
   866	                top_p=top_p,
   867	                max_tokens=max_tokens,
   868	                repetition_penalty=repetition_penalty,
   869	                seed=seed,
   870	            ):
   871	                if captured_globals is None and global_ids:
   872	                    captured_globals = global_ids
   873
   874	                # Crossfade with previous tail (inter-chunk stitching)
   875	                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   876	                    previous_chunk_tail,
   877	                    audio_bytes,
   878	                    sample_rate_hz=sample_rate,
   879	                    crossfade_ms=50,
   880	                )
   881
   882	                if to_emit:
   883	                    yield to_emit, chunk_metrics
   884	        else:
   885	            # Continuation chunks: use captured globals for voice consistency
   886	            if captured_globals is None:
   887	                logger.warning(f"No captured globals for chunk {i+1}, using regular streaming")
   888	                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
   889	                    speaker=speaker,
   890	                    text=chunk_text,
   891	                    temperature=temperature,
   892	                    top_k=top_k,
   893	                    top_p=top_p,
   894	                    max_tokens=max_tokens,
   895	                    repetition_penalty=repetition_penalty,
   896	                    seed=seed,
   897	                ):
   898	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   899	                        previous_chunk_tail,
   900	                        audio_bytes,
   901	                        sample_rate_hz=sample_rate,
   902	                        crossfade_ms=50,
   903	                    )
   904	                    if to_emit:
   905	                        yield to_emit, chunk_metrics
   906	            else:
   907	                # Use continuation method with cached globals
   908	                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
   909	                    speaker=speaker,
   910	                    text=chunk_text,
   911	                    global_ids=captured_globals,
   912	                    temperature=temperature,
   913	                    top_k=top_k,
   914	                    top_p=top_p,
   915	                    max_tokens=max_tokens,
   916	                    repetition_penalty=repetition_penalty,
   917	                    seed=seed,
   918	                ):
   919	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   920	                        previous_chunk_tail,
   921	                        audio_bytes,
   922	                        sample_rate_hz=sample_rate,
   923	                        crossfade_ms=50,
   924	                    )
   925	                    if to_emit:
   926	                        yield to_emit, chunk_metrics
   927
   928	    # Flush remaining tail
   929	    if previous_chunk_tail:
   930	        yield previous_chunk_tail, chunk_metrics


**Patching streaming generator to handle metrics**


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py
@@
         if cached_globals:
             # Fast path: use cached globals, skip global token generation entirely
             stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                 speaker=speaker,
                 text=text,
                 global_ids=cached_globals,
                 temperature=temperature,
                 top_k=top_k,
                 top_p=top_p,
                 max_tokens=max_tokens,
                 repetition_penalty=repetition_penalty,
                 seed=seed,
+                emit_progress=True,
             )
         else:
             # Fallback: no cached globals, generate them inline (original path)
             stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
                 speaker=speaker,
                 text=text,
                 temperature=temperature,
                 top_k=top_k,
                 top_p=top_p,
                 max_tokens=max_tokens,
                 repetition_penalty=repetition_penalty,
                 seed=seed,
+                emit_progress=True,
             )

-        async for audio_chunk in stream_gen:
+        async for stream_item in stream_gen:
+            stream_metrics: Dict[str, Any] = {}
+            audio_chunk = stream_item
+            if (
+                isinstance(stream_item, tuple)
+                and len(stream_item) == 2
+                and isinstance(stream_item[1], dict)
+            ):
+                audio_chunk = stream_item[0]
+                stream_metrics = stream_item[1]
+
+            if stream_metrics:
+                metrics.update(stream_metrics)
+
+            if not audio_chunk:
+                # Metrics-only event from streaming pipeline.
+                continue
+
             # First chunk: prepend WAV header
             if first_chunk_time is None:
                 first_chunk_time = time.time()
                 metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                 wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Preparing runtime patch and inspection**


I’ve wired the streaming pipeline to emit final per-request internal metrics (queue/scheduler/batch/decode timelines) into runtime metrics without changing client-facing audio bytes. Next I’m updating the streaming stress script to aggregate and print these bottleneck stats by concurrency level.


cd /home/ubuntu/veenaModal && nl -ba scripts/stress_test_streaming_runtime.py | sed -n '1,280p'


Output:
Chunk ID: fad576
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2804
Output:
     1	#!/usr/bin/env python3
     2	"""
     3	Direct-runtime streaming stress benchmark for Veena3 TTS.
     4
     5	Focuses on streaming KPIs:
     6	- TTFB (time to first chunk)
     7	- End-to-end latency
     8	- Chunk cadence (inter-chunk interval)
     9	- Emission speed (audio seconds emitted per second after first chunk)
    10	- Throughput under concurrency
    11	"""
    12
    13	from __future__ import annotations
    14
    15	import argparse
    16	import asyncio
    17	import json
    18	import os
    19	import statistics
    20	import subprocess
    21	import sys
    22	import threading
    23	import time
    24	from dataclasses import dataclass, field
    25	from typing import Any, Dict, List, Optional
    26
    27
    28	# Path setup (same pattern as local_server.py)
    29	REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    30	EXTERNAL_DIR = os.path.join(REPO_ROOT, "external")
    31	for path in [os.path.join(EXTERNAL_DIR, "sparktts"), os.path.join(EXTERNAL_DIR, "AP-BWE"), REPO_ROOT]:
    32	    if path not in sys.path:
    33	        sys.path.insert(0, path)
    34
    35	os.environ["AUTH_BYPASS_MODE"] = "true"
    36
    37	TEST_TEXTS = {
    38	    "short": "Hello, this is a quick streaming benchmark for voice agents.",
    39	    "medium": (
    40	        "This streaming benchmark sentence is designed to evaluate first audio latency, "
    41	        "chunk cadence, and sustained emission speed under concurrent request load."
    42	    ),
    43	    "long": (
    44	        "In a bustling city, a narrator collected voice notes from people walking through markets, "
    45	        "subways, and parks. The stream needed to start quickly, stay stable, and deliver clear speech "
    46	        "without stalls as many listeners connected at once."
    47	    ),
    48	}
    49
    50	SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]
    51
    52
    53	@dataclass
    54	class GPUSnapshot:
    55	    timestamp: float
    56	    memory_used_mb: float
    57	    memory_total_mb: float
    58	    gpu_utilization_pct: float
    59
    60
    61	class GPUMonitor:
    62	    def __init__(self, interval_seconds: float = 0.5):
    63	        self.interval = interval_seconds
    64	        self.snapshots: List[GPUSnapshot] = []
    65	        self._running = False
    66	        self._thread: Optional[threading.Thread] = None
    67
    68	    def start(self) -> None:
    69	        self._running = True
    70	        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
    71	        self._thread.start()
    72
    73	    def stop(self) -> None:
    74	        self._running = False
    75	        if self._thread:
    76	            self._thread.join(timeout=3)
    77
    78	    def _poll_loop(self) -> None:
    79	        while self._running:
    80	            try:
    81	                res = subprocess.run(
    82	                    [
    83	                        "nvidia-smi",
    84	                        "--query-gpu=memory.used,memory.total,utilization.gpu",
    85	                        "--format=csv,noheader,nounits",
    86	                    ],
    87	                    capture_output=True,
    88	                    text=True,
    89	                    timeout=5,
    90	                )
    91	                if res.returncode == 0 and res.stdout:
    92	                    parts = [p.strip() for p in res.stdout.strip().split(",")]
    93	                    if len(parts) >= 3:
    94	                        self.snapshots.append(
    95	                            GPUSnapshot(
    96	                                timestamp=time.time(),
    97	                                memory_used_mb=float(parts[0]),
    98	                                memory_total_mb=float(parts[1]),
    99	                                gpu_utilization_pct=float(parts[2]),
   100	                            )
   101	                        )
   102	            except Exception:
   103	                pass
   104	            time.sleep(self.interval)
   105
   106	    def summary(self) -> Dict[str, Any]:
   107	        if not self.snapshots:
   108	            return {"samples": 0, "error": "no GPU samples captured"}
   109	        mem = [s.memory_used_mb for s in self.snapshots]
   110	        util = [s.gpu_utilization_pct for s in self.snapshots]
   111	        return {
   112	            "samples": len(self.snapshots),
   113	            "memory_mb": {
   114	                "min": min(mem),
   115	                "avg": statistics.mean(mem),
   116	                "max": max(mem),
   117	                "total": self.snapshots[0].memory_total_mb,
   118	            },
   119	            "gpu_util_pct": {
   120	                "min": min(util),
   121	                "avg": statistics.mean(util),
   122	                "max": max(util),
   123	            },
   124	        }
   125
   126
   127	@dataclass
   128	class RequestResult:
   129	    success: bool
   130	    latency_ms: float
   131	    ttfb_ms: float
   132	    chunks_sent: int
   133	    total_bytes: int
   134	    audio_seconds: float
   135	    speaker: str
   136	    error: Optional[str] = None
   137	    inter_chunk_ms: List[float] = field(default_factory=list)
   138	    tail_emit_speed_x: float = 0.0
   139	    final_metrics: Dict[str, Any] = field(default_factory=dict)
   140
   141
   142	@dataclass
   143	class LevelResult:
   144	    concurrency: int
   145	    total_requests: int
   146	    results: List[RequestResult] = field(default_factory=list)
   147	    wall_time_s: float = 0.0
   148	    gpu_summary: Dict[str, Any] = field(default_factory=dict)
   149
   150	    @property
   151	    def successes(self) -> List[RequestResult]:
   152	        return [r for r in self.results if r.success]
   153
   154	    @property
   155	    def failures(self) -> List[RequestResult]:
   156	        return [r for r in self.results if not r.success]
   157
   158	    @property
   159	    def success_rate(self) -> float:
   160	        return len(self.successes) / len(self.results) if self.results else 0.0
   161
   162	    @property
   163	    def throughput_rps(self) -> float:
   164	        return self.total_requests / self.wall_time_s if self.wall_time_s > 0 else 0.0
   165
   166	    def _pct(self, vals: List[float], p: int) -> float:
   167	        if not vals:
   168	            return 0.0
   169	        s = sorted(vals)
   170	        idx = int((len(s) - 1) * (p / 100))
   171	        return s[idx]
   172
   173	    def _stats(self, vals: List[float]) -> Dict[str, float]:
   174	        if not vals:
   175	            return {}
   176	        return {
   177	            "avg": statistics.mean(vals),
   178	            "p50": self._pct(vals, 50),
   179	            "p95": self._pct(vals, 95),
   180	            "max": max(vals),
   181	        }
   182
   183	    def latency_stats(self) -> Dict[str, float]:
   184	        return self._stats([r.latency_ms for r in self.successes])
   185
   186	    def ttfb_stats(self) -> Dict[str, float]:
   187	        return self._stats([r.ttfb_ms for r in self.successes if r.ttfb_ms > 0])
   188
   189	    def chunk_count_stats(self) -> Dict[str, float]:
   190	        return self._stats([float(r.chunks_sent) for r in self.successes])
   191
   192	    def emit_speed_stats(self) -> Dict[str, float]:
   193	        return self._stats([r.tail_emit_speed_x for r in self.successes if r.tail_emit_speed_x > 0])
   194
   195	    def inter_chunk_stats(self) -> Dict[str, float]:
   196	        vals: List[float] = []
   197	        for r in self.successes:
   198	            vals.extend(r.inter_chunk_ms)
   199	        return self._stats(vals)
   200
   201
   202	def init_runtime(model_path: str, gpu_mem: float, stream_output_kind: str) -> None:
   203	    from veena3modal.local_server import resolve_model_paths
   204	    from veena3modal.services.tts_runtime import initialize_runtime, is_initialized
   205
   206	    os.environ["VEENA3_STREAM_OUTPUT_KIND"] = stream_output_kind
   207
   208	    llm_path, bicodec_path = resolve_model_paths(model_path)
   209	    if not is_initialized():
   210	        initialize_runtime(
   211	            model_path=llm_path,
   212	            bicodec_path=bicodec_path,
   213	            device="cuda",
   214	            gpu_memory_utilization=gpu_mem,
   215	            precompute_speaker_globals=False,
   216	        )
   217
   218
   219	async def run_one_request(
   220	    text: str,
   221	    speaker: str,
   222	    max_tokens: int,
   223	    timeout_s: float,
   224	    enable_chunking: bool,
   225	) -> RequestResult:
   226	    from veena3modal.services import tts_runtime
   227
   228	    t0 = time.perf_counter()
   229	    first_chunk_t: Optional[float] = None
   230	    prev_chunk_t: Optional[float] = None
   231	    inter_chunk_ms: List[float] = []
   232	    chunks = 0
   233	    total_bytes = 0
   234	    final_metrics: Dict[str, Any] = {}
   235
   236	    async def _consume_stream() -> None:
   237	        nonlocal first_chunk_t, prev_chunk_t, chunks, total_bytes, final_metrics
   238	        gen = tts_runtime.generate_speech_streaming(
   239	            text=text,
   240	            speaker=speaker,
   241	            max_tokens=max_tokens,
   242	            enable_chunking=enable_chunking,
   243	        )
   244	        async for audio_chunk, metrics in gen:
   245	            now = time.perf_counter()
   246	            if first_chunk_t is None:
   247	                first_chunk_t = now
   248	            if prev_chunk_t is not None:
   249	                inter_chunk_ms.append((now - prev_chunk_t) * 1000.0)
   250	            prev_chunk_t = now
   251	            chunks += 1
   252	            total_bytes += len(audio_chunk)
   253	            final_metrics = dict(metrics or {})
   254
   255	    try:
   256	        await asyncio.wait_for(_consume_stream(), timeout=timeout_s)
   257	        end = time.perf_counter()
   258	        latency_ms = (end - t0) * 1000.0
   259	        ttfb_ms = ((first_chunk_t - t0) * 1000.0) if first_chunk_t is not None else 0.0
   260	        audio_seconds = float(final_metrics.get("audio_duration_seconds", 0.0) or 0.0)
   261	        tail_s = max(1e-6, (latency_ms - ttfb_ms) / 1000.0)
   262	        tail_emit_speed_x = audio_seconds / tail_s if audio_seconds > 0 else 0.0
   263	        return RequestResult(
   264	            success=True,
   265	            latency_ms=latency_ms,
   266	            ttfb_ms=ttfb_ms,
   267	            chunks_sent=chunks,
   268	            total_bytes=total_bytes,
   269	            audio_seconds=audio_seconds,
   270	            speaker=speaker,
   271	            inter_chunk_ms=inter_chunk_ms,
   272	            tail_emit_speed_x=tail_emit_speed_x,
   273	            final_metrics=final_metrics,
   274	        )
   275	    except asyncio.TimeoutError:
   276	        return RequestResult(
   277	            success=False,
   278	            latency_ms=(time.perf_counter() - t0) * 1000.0,
   279	            ttfb_ms=0.0,
   280	            chunks_sent=chunks,


**Planning line handling**


cd /home/ubuntu/veenaModal && nl -ba scripts/stress_test_streaming_runtime.py | sed -n '280,520p'


Output:
Chunk ID: 5e3029
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2336
Output:
   280	            chunks_sent=chunks,
   281	            total_bytes=total_bytes,
   282	            audio_seconds=0.0,
   283	            speaker=speaker,
   284	            error="timeout",
   285	            inter_chunk_ms=inter_chunk_ms,
   286	            final_metrics=final_metrics,
   287	        )
   288	    except Exception as exc:
   289	        return RequestResult(
   290	            success=False,
   291	            latency_ms=(time.perf_counter() - t0) * 1000.0,
   292	            ttfb_ms=0.0,
   293	            chunks_sent=chunks,
   294	            total_bytes=total_bytes,
   295	            audio_seconds=0.0,
   296	            speaker=speaker,
   297	            error=str(exc)[:200],
   298	            inter_chunk_ms=inter_chunk_ms,
   299	            final_metrics=final_metrics,
   300	        )
   301
   302
   303	async def run_level(
   304	    concurrency: int,
   305	    total_requests: int,
   306	    text: str,
   307	    max_tokens: int,
   308	    timeout_s: float,
   309	    enable_chunking: bool,
   310	    gpu_monitor: GPUMonitor,
   311	) -> LevelResult:
   312	    level = LevelResult(concurrency=concurrency, total_requests=total_requests)
   313	    semaphore = asyncio.Semaphore(concurrency)
   314
   315	    async def wrapped(i: int) -> RequestResult:
   316	        async with semaphore:
   317	            speaker = SPEAKERS[i % len(SPEAKERS)]
   318	            return await run_one_request(
   319	                text=text,
   320	                speaker=speaker,
   321	                max_tokens=max_tokens,
   322	                timeout_s=timeout_s,
   323	                enable_chunking=enable_chunking,
   324	            )
   325
   326	    gpu_monitor.snapshots.clear()
   327	    t0 = time.perf_counter()
   328	    level.results = await asyncio.gather(*[wrapped(i) for i in range(total_requests)])
   329	    level.wall_time_s = time.perf_counter() - t0
   330	    level.gpu_summary = gpu_monitor.summary()
   331	    return level
   332
   333
   334	def print_level(level: LevelResult) -> None:
   335	    print(f"\n=== streaming concurrency={level.concurrency} requests={level.total_requests} ===")
   336	    print(f"success={len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
   337	    print(f"wall={level.wall_time_s:.2f}s throughput={level.throughput_rps:.2f} req/s")
   338	    lat = level.latency_stats()
   339	    if lat:
   340	        print(f"latency_ms avg={lat['avg']:.0f} p50={lat['p50']:.0f} p95={lat['p95']:.0f} max={lat['max']:.0f}")
   341	    ttfb = level.ttfb_stats()
   342	    if ttfb:
   343	        print(f"ttfb_ms avg={ttfb['avg']:.0f} p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}")
   344	    cc = level.chunk_count_stats()
   345	    if cc:
   346	        print(f"chunks_per_req avg={cc['avg']:.1f} p50={cc['p50']:.1f} p95={cc['p95']:.1f} max={cc['max']:.1f}")
   347	    ic = level.inter_chunk_stats()
   348	    if ic:
   349	        print(
   350	            "inter_chunk_ms avg={:.1f} p50={:.1f} p95={:.1f} max={:.1f}".format(
   351	                ic["avg"], ic["p50"], ic["p95"], ic["max"]
   352	            )
   353	        )
   354	    es = level.emit_speed_stats()
   355	    if es:
   356	        print(
   357	            "tail_emit_speed_x avg={:.2f} p50={:.2f} p95={:.2f} max={:.2f}".format(
   358	                es["avg"], es["p50"], es["p95"], es["max"]
   359	            )
   360	        )
   361	    if "memory_mb" in level.gpu_summary:
   362	        g = level.gpu_summary
   363	        print(
   364	            "gpu: util_avg={:.0f}% util_max={:.0f}% mem_avg={:.0f}MB mem_max={:.0f}MB".format(
   365	                g["gpu_util_pct"]["avg"],
   366	                g["gpu_util_pct"]["max"],
   367	                g["memory_mb"]["avg"],
   368	                g["memory_mb"]["max"],
   369	            )
   370	        )
   371
   372
   373	def save_results(path: str, levels: List[LevelResult], args: argparse.Namespace) -> None:
   374	    payload = {
   375	        "config": {
   376	            "levels": args.levels,
   377	            "text": args.text,
   378	            "max_tokens": args.max_tokens,
   379	            "timeout_s": args.timeout,
   380	            "requests_multiplier": args.requests_multiplier,
   381	            "min_requests": args.min_requests,
   382	            "gpu_memory": args.gpu_memory,
   383	            "stream_output_kind": args.stream_output_kind,
   384	            "chunking": args.chunking,
   385	            "timestamp": int(time.time()),
   386	        },
   387	        "levels": [],
   388	    }
   389	    for lvl in levels:
   390	        payload["levels"].append(
   391	            {
   392	                "concurrency": lvl.concurrency,
   393	                "total_requests": lvl.total_requests,
   394	                "success_rate": lvl.success_rate,
   395	                "throughput_rps": lvl.throughput_rps,
   396	                "wall_time_s": lvl.wall_time_s,
   397	                "latency_ms": lvl.latency_stats(),
   398	                "ttfb_ms": lvl.ttfb_stats(),
   399	                "chunks_per_req": lvl.chunk_count_stats(),
   400	                "inter_chunk_ms": lvl.inter_chunk_stats(),
   401	                "tail_emit_speed_x": lvl.emit_speed_stats(),
   402	                "gpu": lvl.gpu_summary,
   403	                "requests": [
   404	                    {
   405	                        "success": r.success,
   406	                        "latency_ms": r.latency_ms,
   407	                        "ttfb_ms": r.ttfb_ms,
   408	                        "chunks_sent": r.chunks_sent,
   409	                        "total_bytes": r.total_bytes,
   410	                        "audio_seconds": r.audio_seconds,
   411	                        "speaker": r.speaker,
   412	                        "error": r.error,
   413	                        "inter_chunk_ms": r.inter_chunk_ms,
   414	                        "tail_emit_speed_x": r.tail_emit_speed_x,
   415	                        "final_metrics": r.final_metrics,
   416	                    }
   417	                    for r in lvl.results
   418	                ],
   419	            }
   420	        )
   421	    with open(path, "w", encoding="utf-8") as fd:
   422	        json.dump(payload, fd, indent=2)
   423
   424
   425	async def main_async(args: argparse.Namespace) -> None:
   426	    text = TEST_TEXTS[args.text]
   427	    levels = [int(x.strip()) for x in args.levels.split(",") if x.strip()]
   428	    gpu_monitor = GPUMonitor(interval_seconds=0.5)
   429	    gpu_monitor.start()
   430
   431	    # warmup
   432	    await run_one_request(
   433	        text=text,
   434	        speaker=SPEAKERS[0],
   435	        max_tokens=args.max_tokens,
   436	        timeout_s=args.timeout,
   437	        enable_chunking=args.chunking,
   438	    )
   439
   440	    results: List[LevelResult] = []
   441	    for idx, c in enumerate(levels):
   442	        n = max(args.min_requests, c * args.requests_multiplier)
   443	        level = await run_level(
   444	            concurrency=c,
   445	            total_requests=n,
   446	            text=text,
   447	            max_tokens=args.max_tokens,
   448	            timeout_s=args.timeout,
   449	            enable_chunking=args.chunking,
   450	            gpu_monitor=gpu_monitor,
   451	        )
   452	        print_level(level)
   453	        results.append(level)
   454	        if idx < len(levels) - 1:
   455	            await asyncio.sleep(2)
   456
   457	    gpu_monitor.stop()
   458	    save_results(args.output, results, args)
   459	    print(f"\nresults saved: {args.output}")
   460
   461
   462	def main() -> None:
   463	    parser = argparse.ArgumentParser(description="Direct runtime streaming stress benchmark")
   464	    parser.add_argument("--levels", default="16,32,64", help="Comma-separated concurrency levels")
   465	    parser.add_argument("--text", choices=["short", "medium", "long"], default="short")
   466	    parser.add_argument("--max-tokens", type=int, default=512, help="max_tokens sent to streaming generation")
   467	    parser.add_argument("--timeout", type=float, default=180.0, help="Per-request timeout seconds")
   468	    parser.add_argument("--requests-multiplier", type=int, default=2, help="Requests per level ~= concurrency * multiplier")
   469	    parser.add_argument("--min-requests", type=int, default=8)
   470	    parser.add_argument("--chunking", action="store_true", default=False, help="Enable long-text chunking path")
   471	    parser.add_argument("--gpu-memory", type=float, default=0.25)
   472	    parser.add_argument("--stream-output-kind", choices=["delta", "cumulative"], default="delta")
   473	    parser.add_argument("--model-path", type=str, default="")
   474	    parser.add_argument("--output", default="stress_streaming_runtime.json")
   475	    args = parser.parse_args()
   476
   477	    from veena3modal.local_server import DEFAULT_LOCAL_MODEL_DIR
   478
   479	    model_path = args.model_path or DEFAULT_LOCAL_MODEL_DIR
   480	    init_runtime(model_path=model_path, gpu_mem=args.gpu_memory, stream_output_kind=args.stream_output_kind)
   481	    asyncio.run(main_async(args))
   482
   483
   484	if __name__ == "__main__":
   485	    main()


**Adding timing stats methods**


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/scripts/stress_test_streaming_runtime.py
@@
 class LevelResult:
@@
     def inter_chunk_stats(self) -> Dict[str, float]:
         vals: List[float] = []
         for r in self.successes:
             vals.extend(r.inter_chunk_ms)
         return self._stats(vals)
+
+    def timing_stats(self, key: str) -> Dict[str, float]:
+        vals: List[float] = []
+        for r in self.successes:
+            v = r.final_metrics.get(key)
+            if isinstance(v, (int, float)):
+                vals.append(float(v))
+        return self._stats(vals)
+
+    def latency_over_metric_stats(self, key: str) -> Dict[str, float]:
+        vals: List[float] = []
+        for r in self.successes:
+            base = r.final_metrics.get(key)
+            if isinstance(base, (int, float)):
+                vals.append(max(0.0, float(r.latency_ms) - float(base)))
+        return self._stats(vals)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Updating runtime initialization**


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/scripts/stress_test_streaming_runtime.py
@@
-def init_runtime(model_path: str, gpu_mem: float, stream_output_kind: str) -> None:
+def init_runtime(
+    model_path: str,
+    gpu_mem: float,
+    stream_output_kind: str,
+    num_engines: int,
+    max_num_batched_tokens: Optional[int],
+    max_num_seqs: Optional[int],
+    enable_chunked_prefill: Optional[bool],
+    enable_prefix_caching: Optional[bool],
+    disable_log_stats: Optional[bool],
+    enforce_eager: Optional[bool],
+) -> None:
@@
     if not is_initialized():
         initialize_runtime(
             model_path=llm_path,
             bicodec_path=bicodec_path,
             device="cuda",
             gpu_memory_utilization=gpu_mem,
+            num_engines=num_engines,
+            max_num_batched_tokens=max_num_batched_tokens,
+            max_num_seqs=max_num_seqs,
+            enable_chunked_prefill=enable_chunked_prefill,
+            enable_prefix_caching=enable_prefix_caching,
+            disable_log_stats=disable_log_stats,
+            enforce_eager=enforce_eager,
             precompute_speaker_globals=False,
         )
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Enhancing timing details in output**


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/scripts/stress_test_streaming_runtime.py
@@
 def print_level(level: LevelResult) -> None:
@@
     es = level.emit_speed_stats()
     if es:
         print(
             "tail_emit_speed_x avg={:.2f} p50={:.2f} p95={:.2f} max={:.2f}".format(
                 es["avg"], es["p50"], es["p95"], es["max"]
             )
         )
+
+    timing_keys = [
+        "timeline_to_first_batch_ms",
+        "timeline_to_first_chunk_emitted_ms",
+        "timeline_total_ms",
+        "llm_time_in_queue_ms",
+        "llm_scheduler_ms",
+        "llm_first_token_ms",
+        "llm_request_lifecycle_ms",
+        "llm_generation_wall_ms",
+        "llm_time_per_token_ms",
+        "llm_batch_wall_ms_total",
+        "llm_decode_wall_ms_total",
+        "tokens_per_batch_p50",
+        "bicodec_decode_calls",
+        "bicodec_decode_wall_ms_total",
+        "bicodec_decode_interval_ms_p50",
+    ]
+    for key in timing_keys:
+        st = level.timing_stats(key)
+        if st:
+            print(f"{key}: avg={st['avg']:.2f} p50={st['p50']:.2f} p95={st['p95']:.2f} max={st['max']:.2f}")
+
+    latency_over_timeline = level.latency_over_metric_stats("timeline_total_ms")
+    if latency_over_timeline:
+        print(
+            "latency_over_timeline_ms: avg={:.2f} p50={:.2f} p95={:.2f} max={:.2f}".format(
+                latency_over_timeline["avg"],
+                latency_over_timeline["p50"],
+                latency_over_timeline["p95"],
+                latency_over_timeline["max"],
+            )
+        )
+
+    tl_total = level.timing_stats("timeline_total_ms")
+    llm_total = level.timing_stats("llm_generation_wall_ms")
+    bicodec_total = level.timing_stats("bicodec_decode_wall_ms_total")
+    if tl_total and llm_total:
+        total_avg = tl_total.get("avg", 0.0)
+        llm_avg = llm_total.get("avg", 0.0)
+        bicodec_avg = bicodec_total.get("avg", 0.0) if bicodec_total else 0.0
+        other_avg = max(0.0, total_avg - llm_avg - bicodec_avg)
+        if total_avg > 0:
+            print(
+                "timeline_share: llm={:.1f}% bicodec={:.1f}% other={:.1f}%".format(
+                    (llm_avg / total_avg) * 100,
+                    (bicodec_avg / total_avg) * 100,
+                    (other_avg / total_avg) * 100,
+                )
+            )
+
     if "memory_mb" in level.gpu_summary:
         g = level.gpu_summary
         print(
             "gpu: util_avg={:.0f}% util_max={:.0f}% mem_avg={:.0f}MB mem_max={:.0f}MB".format(
                 g["gpu_util_pct"]["avg"],
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Patching result saving and timing**