Input:
*** Begin Patch
*** Update File: veena3modal/core/streaming_pipeline.py
@@
     async def generate_speech_stream_indic_first_chunk(
         self,
         speaker: str,
         text: str,
@@
     ) -> AsyncGenerator[tuple, None]:
@@
-        import time
-        t_start = time.time()
-        request_metrics = request_metrics or {}
-        request_metrics.setdefault("request_id", request_id)
+        import time
+        import torch
+
+        t_start = time.time()
+        emit_timing = _detailed_timing_enabled()
+        request_metrics = request_metrics or {}
+        request_metrics.update(
+            {
+                "request_id": request_id,
+                "speaker": speaker,
+                "text_length": len(text),
+                "text_chunked": True,
+            }
+        )
+
+        batch_wall_ms: List[float] = []
+        batch_token_counts: List[int] = []
+        batch_gpu_ms: List[float] = []
+        parse_wall_ms_list: List[float] = []
+        decode_calls = 0
+        decode_wall_ms = 0.0
+        decode_gpu_ms = 0.0
+        decode_wall_samples: List[float] = []
+        decode_gpu_samples: List[float] = []
@@
-        results_generator = self.model.engine.generate(
+        results_generator = self.model.engine.generate(
             prompt=prompt,
             sampling_params=sampling_params,
             request_id=request_id,
         )

         # Track state
         last_decode_count = 0
         total_samples_emitted_to_user = 0
         previous_chunk_tail: Optional[bytes] = None
         captured_globals: List[int] = []  # Will be populated once we have 32 globals
+        t_first_token = None
+        t_first_chunk_ready = None
+        t_first_chunk_decoded = None
+        iteration_count = 0

         # Stream tokens with BiCodec extraction
         async for request_output in results_generator:
+            iteration_count += 1
+            t_iter = time.time()
+
+            if t_first_token is None:
+                t_first_token = t_iter
+
             generated_ids = request_output.outputs[0].token_ids
-
-            # OPTIMIZATION: Incremental parsing (O(1) per new token)
-            new_token_ids = generated_ids[processed_token_count:]
-            processed_token_count = len(generated_ids)
-            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
-
+
+            parse_start = time.time()
+            # OPTIMIZATION: Incremental parsing (O(1) per new token)
+            new_token_ids = generated_ids[processed_token_count:]
+            processed_token_count = len(generated_ids)
+            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
+            parse_wall_ms = (time.time() - parse_start) * 1000
+            parse_wall_ms_list.append(parse_wall_ms)
+            batch_token_counts.append(len(new_token_ids))
+            batch_wall_ms_value = (time.time() - t_iter) * 1000 if t_iter else 0.0
+            batch_wall_ms.append(batch_wall_ms_value)
+            if emit_timing:
+                logger.info(
+                    "tts_stream_llm_batch",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "batch_index": iteration_count,
+                        "batch_tokens": len(new_token_ids),
+                        "llm_batch_wall_ms": int(batch_wall_ms_value),
+                        "llm_batch_gpu_ms": int(batch_gpu_ms[-1]) if batch_gpu_ms else None,
+                        "text_chunked": True,
+                    },
+                )
+
             # Capture global tokens once we have all 32
             if not captured_globals and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                 captured_globals = global_buffer[:EXPECTED_GLOBAL_COUNT]

             # Process if we have all global tokens
             if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
+                if t_first_chunk_ready is None and len(semantic_buffer) >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
+                    t_first_chunk_ready = time.time()
                 semantic_count = len(semantic_buffer)
                 if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                     if semantic_count - last_decode_count >= DECODE_INTERVAL:
                         last_decode_count = semantic_count
@@
-                        if self.snac_decoder.enable_batching:
+                        decode_gpu_ms_this = None
+                        t_decode_start = time.time()
+                        if emit_timing and torch.cuda.is_available():
+                            torch.cuda.synchronize()
+                            decode_start_event = torch.cuda.Event(enable_timing=True)
+                            decode_end_event = torch.cuda.Event(enable_timing=True)
+                            decode_start_event.record()
+
+                        if self.snac_decoder.enable_batching:
                             audio_bytes = await self.snac_decoder.decode_single_async(
                                 semantic_ids=all_semantic,
                                 global_ids=window_global,
                                 trim_warmup=False,
                                 use_sliding_window=False
                             )
                         else:
                             audio_bytes = self.snac_decoder.decode_streaming(
                                 semantic_ids=all_semantic,
                                 global_ids=window_global,
                                 use_sliding_window=False,
                                 trim_warmup=False
                             )
-
-                        if audio_bytes:
+
+                        t_decode_end = time.time()
+                        decode_calls += 1
+                        decode_ms = (t_decode_end - t_decode_start) * 1000
+                        decode_wall_ms += decode_ms
+                        decode_wall_samples.append(decode_ms)
+
+                        if emit_timing and torch.cuda.is_available():
+                            decode_end_event.record()
+                            torch.cuda.synchronize()
+                            decode_gpu_ms_this = decode_start_event.elapsed_time(decode_end_event)
+                            decode_gpu_ms += decode_gpu_ms_this
+                            decode_gpu_samples.append(decode_gpu_ms_this)
+                            batch_gpu_ms.append(decode_gpu_ms_this)
+
+                        to_emit = b""
+                        if audio_bytes:
                             total_samples_decoded = len(audio_bytes) // 2

                             if total_samples_decoded > total_samples_emitted_to_user:
                                 new_bytes_start = total_samples_emitted_to_user * 2
                                 new_audio_bytes = audio_bytes[new_bytes_start:]

                                 to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                     previous_chunk_tail,
                                     new_audio_bytes,
                                     sample_rate_hz=16000,
                                     crossfade_ms=CROSSFADE_MS,
                                 )

                                 samples_emitted_in_this_chunk = len(to_emit) // 2
                                 total_samples_emitted_to_user += samples_emitted_in_this_chunk
+
+                                if to_emit:
+                                    total_audio_chunks += 1
+                                    if t_first_chunk_decoded is None:
+                                        t_first_chunk_decoded = t_decode_end
+
+                                    if emit_timing:
+                                        logger.info(
+                                            "tts_stream_decode_batch",
+                                            extra={
+                                                "speaker": speaker,
+                                                "request_id": request_id,
+                                                "batch_index": decode_calls,
+                                                "decode_wall_ms": int(decode_ms),
+                                                "decode_gpu_ms": int(decode_gpu_ms_this)
+                                                if decode_gpu_ms_this is not None else None,
+                                                "semantic_count": len(semantic_buffer),
+                                                "text_chunked": True,
+                                            },
+                                        )
+
+                                    yield (to_emit, captured_globals)
-
-                                if to_emit:
-                                    total_audio_chunks += 1
-                                    # Yield tuple: (audio_bytes, global_ids)
-                                    yield (to_emit, captured_globals)
-
+
         # Final chunk processing
         if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
+            t_decode_start = time.time()
             audio_bytes = self.snac_decoder.decode_streaming(
                 semantic_ids=semantic_buffer,
                 global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
                 use_sliding_window=False,
                 trim_warmup=False
             )
-            if audio_bytes:
+            t_decode_end = time.time()
+            decode_ms = (t_decode_end - t_decode_start) * 1000
+            decode_calls += 1
+            decode_wall_ms += decode_ms
+            decode_wall_samples.append(decode_ms)
+
+            to_emit = b""
+            if audio_bytes:
                 total_samples_decoded = len(audio_bytes) // 2

                 if total_samples_decoded > total_samples_emitted_to_user:
                     new_bytes_start = total_samples_emitted_to_user * 2
                     new_audio_bytes = audio_bytes[new_bytes_start:]
@@
                     to_emit, previous_chunk_tail = crossfade_bytes_int16(
                         previous_chunk_tail,
                         new_audio_bytes,
                         sample_rate_hz=16000,
                         crossfade_ms=CROSSFADE_MS,
                     )

                     samples_emitted_in_final = len(to_emit) // 2
                     total_samples_emitted_to_user += samples_emitted_in_final
-
-                        if to_emit:
-                            total_audio_chunks += 1
-                            yield (to_emit, captured_globals)
+                    if t_first_chunk_decoded is None:
+                        t_first_chunk_decoded = t_decode_end
+                    if to_emit:
+                        total_audio_chunks += 1
+                        if emit_timing:
+                            logger.info(
+                                "tts_stream_decode_batch",
+                                extra={
+                                    "speaker": speaker,
+                                    "request_id": request_id,
+                                    "batch_index": decode_calls,
+                                    "decode_wall_ms": int(decode_ms),
+                                    "decode_gpu_ms": None,
+                                    "semantic_count": len(semantic_buffer),
+                                    "text_chunked": True,
+                                },
+                            )
+                        yield (to_emit, captured_globals)

         # Flush remaining tail
         if previous_chunk_tail:
             total_audio_chunks += 1
             tail_samples = len(previous_chunk_tail) // 2
             total_samples_emitted_to_user += tail_samples
             yield (previous_chunk_tail, captured_globals)

         t_end = time.time()
         audio_duration_s = total_samples_emitted_to_user / 16000
+        ttfb_ms = (
+            (t_first_chunk_decoded - t_start) * 1000
+            if t_first_chunk_decoded is not None
+            else (t_first_chunk_ready - t_start) * 1000
+            if t_first_chunk_ready is not None
+            else (t_first_token - t_start) * 1000
+            if t_first_token is not None
+            else 0
+        )
+        total_wall_ms = (t_end - t_start) * 1000
         request_metrics.update(
             {
                 "request_id": request_id,
                 "speaker": speaker,
+                "generated_tokens": len(semantic_buffer) + len(global_buffer),
+                "semantic_tokens": len(semantic_buffer),
+                "global_tokens": len(captured_globals),
+                "llm_batch_count": len(batch_wall_ms),
+                "llm_batch_tokens": batch_token_counts,
+                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
+                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
+                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_parse_ms": int(sum(parse_wall_ms_list)),
+                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
+                "decode_calls": decode_calls,
+                "llm_decode_wall_ms_total": int(decode_wall_ms),
+                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
+                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "llm_decode_gpu_ms_p50": int(statistics.median(decode_gpu_samples)) if decode_gpu_samples else 0,
                 "audio_duration_seconds": audio_duration_s,
                 "audio_chunks": total_audio_chunks,
-                "text_chunked": True,
+                "text_chunked": True,
                 "captured_global_count": len(captured_globals),
-                "semantic_tokens": len(semantic_buffer),
-                "generation_ms": int((t_end - t_start) * 1000),
+                "ttfb_ms": int(ttfb_ms),
+                "generation_ms": int(total_wall_ms),
             }
         )
-        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)
+        if tokens_per_batch := request_metrics.get("llm_batch_tokens"):
+            request_metrics["tokens_per_batch_min"] = int(min(tokens_per_batch))
+            request_metrics["tokens_per_batch_max"] = int(max(tokens_per_batch))
+            request_metrics["tokens_per_batch_p50"] = int(statistics.median(tokens_per_batch))
+        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)
@@
     ) -> AsyncGenerator[bytes, None]:
@@
-        import time
-        t_start = time.time()
-        request_metrics = request_metrics or {}
-        request_metrics.setdefault("request_id", request_id)
+        import time
+        import torch
+
+        t_start = time.time()
+        emit_timing = _detailed_timing_enabled()
+        request_metrics = request_metrics or {}
+        request_metrics.update(
+            {
+                "request_id": request_id,
+                "speaker": speaker,
+                "text_length": len(text),
+                "text_chunked": True,
+            }
+        )
+
+        batch_wall_ms: List[float] = []
+        batch_token_counts: List[int] = []
+        batch_gpu_ms: List[float] = []
+        parse_wall_ms_list: List[float] = []
+        decode_calls = 0
+        decode_wall_ms = 0.0
+        decode_gpu_ms = 0.0
+        decode_wall_samples: List[float] = []
+        decode_gpu_samples: List[float] = []
@@
-        results_generator = self.model.engine.generate(
+        results_generator = self.model.engine.generate(
             prompt=prompt,
             sampling_params=sampling_params,
             request_id=request_id,
         )

         # Track state
         last_decode_count = 0
         total_samples_emitted_to_user = 0
         previous_chunk_tail: Optional[bytes] = None
+        t_first_token = None
+        t_first_chunk_ready = None
+        t_first_chunk_decoded = None
+        iteration_count = 0
+        global_count = len(global_ids)

         # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled in prompt)
         async for request_output in results_generator:
+            iteration_count += 1
+            t_iter = time.time()
+
+            if t_first_token is None:
+                t_first_token = t_iter
+
             generated_ids = request_output.outputs[0].token_ids
-
+
             # OPTIMIZATION: Incremental parsing (O(1) per new token)
             # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
             new_token_ids = generated_ids[processed_token_count:]
             processed_token_count = len(generated_ids)
+            parse_start = time.time()
             token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)
+            parse_wall_ms = (time.time() - parse_start) * 1000
+            parse_wall_ms_list.append(parse_wall_ms)
+            batch_token_counts.append(len(new_token_ids))
+            batch_wall_ms_value = (time.time() - t_iter) * 1000 if t_iter else 0.0
+            batch_wall_ms.append(batch_wall_ms_value)
+            if emit_timing:
+                logger.info(
+                    "tts_stream_llm_batch",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "batch_index": iteration_count,
+                        "batch_tokens": len(new_token_ids),
+                        "llm_batch_wall_ms": int(batch_wall_ms_value),
+                        "llm_batch_gpu_ms": int(batch_gpu_ms[-1]) if batch_gpu_ms else None,
+                        "text_chunked": True,
+                    },
+                )

             semantic_count = len(semantic_buffer)
             if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
+                if t_first_chunk_ready is None:
+                    t_first_chunk_ready = time.time()
                 if semantic_count - last_decode_count >= DECODE_INTERVAL:
                     last_decode_count = semantic_count

                     # Use pre-captured global tokens for decoding
+                    decode_gpu_ms_this = None
+                    t_decode_start = time.time()
+                    if emit_timing and torch.cuda.is_available():
+                        torch.cuda.synchronize()
+                        decode_start_event = torch.cuda.Event(enable_timing=True)
+                        decode_end_event = torch.cuda.Event(enable_timing=True)
+                        decode_start_event.record()
                     if self.snac_decoder.enable_batching:
                         audio_bytes = await self.snac_decoder.decode_single_async(
                             semantic_ids=semantic_buffer,
                             global_ids=global_ids,  # Use captured globals
                             trim_warmup=False,
@@
                             global_ids=global_ids,  # Use captured globals
                             use_sliding_window=False,
                             trim_warmup=False
                         )
-
+
+                    t_decode_end = time.time()
+                    decode_calls += 1
+                    decode_ms = (t_decode_end - t_decode_start) * 1000
+                    decode_wall_ms += decode_ms
+                    decode_wall_samples.append(decode_ms)
+                    if emit_timing and torch.cuda.is_available():
+                        decode_end_event.record()
+                        torch.cuda.synchronize()
+                        decode_gpu_ms_this = decode_start_event.elapsed_time(decode_end_event)
+                        decode_gpu_ms += decode_gpu_ms_this
+                        decode_gpu_samples.append(decode_gpu_ms_this)
+                        batch_gpu_ms.append(decode_gpu_ms_this)
+
+                    to_emit = b""
                     if audio_bytes:
                         total_samples_decoded = len(audio_bytes) // 2

                         if total_samples_decoded > total_samples_emitted_to_user:
                             new_bytes_start = total_samples_emitted_to_user * 2
                             new_audio_bytes = audio_bytes[new_bytes_start:]
@@
                             to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                 previous_chunk_tail,
                                 new_audio_bytes,
                                 sample_rate_hz=16000,
                                 crossfade_ms=CROSSFADE_MS,
                             )

                             samples_emitted_in_this_chunk = len(to_emit) // 2
                             total_samples_emitted_to_user += samples_emitted_in_this_chunk

-                        if to_emit:
-                            total_audio_chunks += 1
-                            yield to_emit
+                            if to_emit:
+                                total_audio_chunks += 1
+                                if t_first_chunk_decoded is None:
+                                    t_first_chunk_decoded = t_decode_end
+
+                                if emit_timing:
+                                    logger.info(
+                                        "tts_stream_decode_batch",
+                                        extra={
+                                            "speaker": speaker,
+                                            "request_id": request_id,
+                                            "batch_index": decode_calls,
+                                            "decode_wall_ms": int(decode_ms),
+                                            "decode_gpu_ms": int(decode_gpu_ms_this)
+                                            if decode_gpu_ms_this is not None else None,
+                                            "semantic_count": len(semantic_buffer),
+                                            "text_chunked": True,
+                                        },
+                                    )
+                                yield to_emit

         # Final chunk processing
         if len(semantic_buffer) > last_decode_count:
+            t_decode_start = time.time()
             audio_bytes = self.snac_decoder.decode_streaming(
                 semantic_ids=semantic_buffer,
                 global_ids=global_ids,  # Use captured globals
                 use_sliding_window=False,
                 trim_warmup=False
             )
+            t_decode_end = time.time()
+            decode_ms = (t_decode_end - t_decode_start) * 1000
+            decode_calls += 1
+            decode_wall_ms += decode_ms
+            decode_wall_samples.append(decode_ms)
+
+            to_emit = b""
             if audio_bytes:
                 total_samples_decoded = len(audio_bytes) // 2

                 if total_samples_decoded > total_samples_emitted_to_user:
                     new_bytes_start = total_samples_emitted_to_user * 2
                     new_audio_bytes = audio_bytes[new_bytes_start:]
@@
                     to_emit, previous_chunk_tail = crossfade_bytes_int16(
                         previous_chunk_tail,
                         new_audio_bytes,
                         sample_rate_hz=16000,
                         crossfade_ms=CROSSFADE_MS,
                     )

                     samples_emitted_in_final = len(to_emit) // 2
                     total_samples_emitted_to_user += samples_emitted_in_final
-
                     if to_emit:
                         total_audio_chunks += 1
+                        if t_first_chunk_decoded is None:
+                            t_first_chunk_decoded = t_decode_end
+                        if emit_timing:
+                            logger.info(
+                                "tts_stream_decode_batch",
+                                extra={
+                                    "speaker": speaker,
+                                    "request_id": request_id,
+                                    "batch_index": decode_calls,
+                                    "decode_wall_ms": int(decode_ms),
+                                    "decode_gpu_ms": None,
+                                    "semantic_count": len(semantic_buffer),
+                                    "text_chunked": True,
+                                },
+                            )
                         yield to_emit

         # Flush remaining tail
         if previous_chunk_tail:
             total_audio_chunks += 1
             tail_samples = len(previous_chunk_tail) // 2
@@

         t_end = time.time()
+        ttfb_ms = (
+            (t_first_chunk_decoded - t_start) * 1000
+            if t_first_chunk_decoded is not None
+            else (t_first_chunk_ready - t_start) * 1000
+            if t_first_chunk_ready is not None
+            else (t_first_token - t_start) * 1000
+            if t_first_token is not None
+            else 0
+        )
+        total_wall_ms = (t_end - t_start) * 1000
         request_metrics.update(
             {
                 "request_id": request_id,
                 "speaker": speaker,
+                "generated_tokens": len(semantic_buffer) + global_count,
+                "semantic_tokens": len(semantic_buffer),
+                "global_tokens": len(global_ids),
+                "llm_batch_count": len(batch_wall_ms),
+                "llm_batch_tokens": batch_token_counts,
+                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
+                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
+                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_parse_ms": int(sum(parse_wall_ms_list)),
+                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
+                "decode_calls": decode_calls,
+                "llm_decode_wall_ms_total": int(decode_wall_ms),
+                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
+                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "llm_decode_gpu_ms_p50": int(statistics.median(decode_gpu_samples)) if decode_gpu_samples else 0,
                 "audio_duration_seconds": total_samples_emitted_to_user / 16000,
                 "audio_chunks": total_audio_chunks,
-                "semantic_tokens": len(semantic_buffer),
-                "global_tokens": len(global_ids),
-                "generation_ms": int((t_end - t_start) * 1000),
+                "text_chunked": True,
+                "captured_global_count": len(global_ids),
+                "ttfb_ms": int(ttfb_ms),
+                "generation_ms": int(total_wall_ms),
             }
         )
+        if tokens_per_batch := request_metrics.get("llm_batch_tokens"):
+            request_metrics["tokens_per_batch_min"] = int(min(tokens_per_batch))
+            request_metrics["tokens_per_batch_max"] = int(max(tokens_per_batch))
+            request_metrics["tokens_per_batch_p50"] = int(statistics.median(tokens_per_batch))
         logger.info("tts_stream_continuation_complete", extra=request_metrics)
*** End Patch


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None

        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled in prompt)
        async for request_output in results_generator:
            generated_ids = request_output.outputs[0].token_ids

            # OPTIMIZATION: Incremental parsing (O(1) per new token)
            # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)

            semantic_count = len(semantic_buffer)
            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                if semantic_count - last_decode_count >= DECODE_INTERVAL:
                    last_decode_count = semantic_count

                    # Use pre-captured global tokens for decoding
                    if self.snac_decoder.enable_batching:
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=semantic_buffer,
                            global_ids=global_ids,  # Use captured globals
                            trim_warmup=False,


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '584,820p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 79e0e7
Wall time: 0.0527 seconds
Process exited with code 0
Original token count: 2740
Output:
    async def generate_speech_stream_indic_first_chunk(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        request_id: Optional[str] = None,
        request_metrics: Optional[Dict[str, Any]] = None,
    ) -> AsyncGenerator[tuple, None]:
        """
        Generate speech for the FIRST chunk of multi-chunk text, capturing global tokens.

        This method is specifically for chunked generation: it yields (audio_bytes, global_ids)
        tuples instead of just audio_bytes. The caller captures global_ids from the first
        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.

        Use Case (chunked streaming with voice consistency):
            globals_captured = None
            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
                if globals_captured is None and global_ids:
                    globals_captured = global_ids  # Capture once
                yield audio_bytes
            # Now use globals_captured for subsequent chunks

        Args:
            speaker: Speaker name
            text: First text chunk to synthesize
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max tokens to generate
            repetition_penalty: Prevent repetition
            seed: Random seed for reproducibility

        Yields:
            Tuple of (audio_bytes, global_ids)
            - audio_bytes: Raw PCM audio (int16, 16kHz)
            - global_ids: List of 32 global token IDs (populated after first decode, else empty)

        Thread Safety:
            This method is stateless and thread-safe. Each call creates its own
            request-scoped state. No global state is modified or shared.
        """
        import time
        t_start = time.time()
        request_metrics = request_metrics or {}
        request_metrics.setdefault("request_id", request_id)

        # Build prompt using Indic prompt builder
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,
            seed=seed,
        )

        # BiCodec token buffers
        semantic_buffer = []
        global_buffer = []
        processed_token_count = 0  # Track how many vLLM tokens we've parsed
        total_audio_chunks = 0

        # BiCodec streaming configuration
        EXPECTED_GLOBAL_COUNT = 32
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
        CROSSFADE_MS = 50

        # Generate unique request ID
        import uuid
        if request_id is None:
            request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
            request_metrics["request_id"] = request_id

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser
        if token_parser is None:
            raise RuntimeError("BiCodecTokenParser unavailable for first-chunk streaming")

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None
        captured_globals: List[int] = []  # Will be populated once we have 32 globals

        # Stream tokens with BiCodec extraction
        async for request_output in results_generator:
            generated_ids = request_output.outputs[0].token_ids

            # OPTIMIZATION: Incremental parsing (O(1) per new token)
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)

            # Capture global tokens once we have all 32
            if not captured_globals and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                captured_globals = global_buffer[:EXPECTED_GLOBAL_COUNT]

            # Process if we have all global tokens
            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                semantic_count = len(semantic_buffer)
                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
                        last_decode_count = semantic_count

                        all_semantic = semantic_buffer
                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]

                        if self.snac_decoder.enable_batching:
                            audio_bytes = await self.snac_decoder.decode_single_async(
                                semantic_ids=all_semantic,
                                global_ids=window_global,
                                trim_warmup=False,
                                use_sliding_window=False
                            )
                        else:
                            audio_bytes = self.snac_decoder.decode_streaming(
                                semantic_ids=all_semantic,
                                global_ids=window_global,
                                use_sliding_window=False,
                                trim_warmup=False
                            )

                        if audio_bytes:
                            total_samples_decoded = len(audio_bytes) // 2

                            if total_samples_decoded > total_samples_emitted_to_user:
                                new_bytes_start = total_samples_emitted_to_user * 2
                                new_audio_bytes = audio_bytes[new_bytes_start:]

                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                    previous_chunk_tail,
                                    new_audio_bytes,
                                    sample_rate_hz=16000,
                                    crossfade_ms=CROSSFADE_MS,
                                )

                                samples_emitted_in_this_chunk = len(to_emit) // 2
                                total_samples_emitted_to_user += samples_emitted_in_this_chunk

                                if to_emit:
                                    total_audio_chunks += 1
                                    # Yield tuple: (audio_bytes, global_ids)
                                    yield (to_emit, captured_globals)

        # Final chunk processing
        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,
                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
                use_sliding_window=False,
                trim_warmup=False
            )
            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    samples_emitted_in_final = len(to_emit) // 2
                    total_samples_emitted_to_user += samples_emitted_in_final

                        if to_emit:
                            total_audio_chunks += 1
                            yield (to_emit, captured_globals)

        # Flush remaining tail
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            yield (previous_chunk_tail, captured_globals)

        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000
        request_metrics.update(
            {
                "request_id": request_id,
                "speaker": speaker,
                "audio_duration_seconds": audio_duration_s,
                "audio_chunks": total_audio_chunks,
                "text_chunked": True,
                "captured_global_count": len(captured_globals),
                "semantic_tokens": len(semantic_buffer),
                "generation_ms": int((t_end - t_start) * 1000),
            }
        )
        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)

    async def generate_speech_stream_indic_continuation(
        self,
        speaker: str,
        text: str,
        global_ids: List[int],
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        request_id: Optional[str] = None,
        request_metrics: Optional[Dict[str, Any]] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech for CONTINUATION chunks using pre-captured global tokens.

        This is the key method for voice consistency in chunked generation:
        - Uses global_ids from the first chunk to maintain identical voice
        - Model generates only semantic tokens (skips global token generation)
        - Ensures no voice drift across chunks


Summary: auto


sed -n '820,1080p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 4af63f
Wall time: 0.0549 seconds
Process exited with code 0
Original token count: 2167
Output:
        - Ensures no voice drift across chunks

        CRITICAL for production:
        - global_ids is request-scoped, passed explicitly (no shared state)
        - Thread-safe: each request has its own state
        - No global caching that could cause cross-request contamination

        Args:
            speaker: Speaker name (must match first chunk)
            text: Continuation text chunk to synthesize
            global_ids: 32 global token IDs captured from first chunk
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max tokens to generate
            repetition_penalty: Prevent repetition
            seed: Random seed (use same as first chunk for consistency)

        Yields:
            Audio bytes (int16 PCM, 16kHz mono)

        Raises:
            ValueError: If global_ids doesn't contain exactly 32 tokens

        Thread Safety:
            Fully thread-safe. All state is request-scoped and passed explicitly.
        """
        import time
        t_start = time.time()
        request_metrics = request_metrics or {}
        request_metrics.setdefault("request_id", request_id)

        # Validate global tokens
        EXPECTED_GLOBAL_COUNT = 32
        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
            raise ValueError(
                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
                f"global_ids must be captured from first chunk generation."
            )

        # Build prompt WITH pre-filled global tokens
        # This tells the model to skip global generation and go straight to semantic tokens
        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)

        # Configure sampling
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,
            seed=seed,
        )

        # BiCodec token buffers
        semantic_buffer = []
        global_buffer_unused = []  # Not used in continuation, but needed for parser API
        processed_token_count = 0  # Track how many vLLM tokens we've parsed
        total_audio_chunks = 0

        # BiCodec streaming configuration
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
        CROSSFADE_MS = 50

        # Generate unique request ID
        import uuid
        if request_id is None:
            request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
            request_metrics["request_id"] = request_id

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser
        if token_parser is None:
            raise RuntimeError("BiCodecTokenParser unavailable for continuation streaming")

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None

        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
        async for request_output in results_generator:
            generated_ids = request_output.outputs[0].token_ids

            # OPTIMIZATION: Incremental parsing (O(1) per new token)
            # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)

            semantic_count = len(semantic_buffer)
            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                if semantic_count - last_decode_count >= DECODE_INTERVAL:
                    last_decode_count = semantic_count

                    # Use pre-captured global tokens for decoding
                    if self.snac_decoder.enable_batching:
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=semantic_buffer,
                            global_ids=global_ids,  # Use captured globals
                            trim_warmup=False,
                            use_sliding_window=False
                        )
                    else:
                        audio_bytes = self.snac_decoder.decode_streaming(
                            semantic_ids=semantic_buffer,
                            global_ids=global_ids,  # Use captured globals
                            use_sliding_window=False,
                            trim_warmup=False
                        )

                    if audio_bytes:
                        total_samples_decoded = len(audio_bytes) // 2

                        if total_samples_decoded > total_samples_emitted_to_user:
                            new_bytes_start = total_samples_emitted_to_user * 2
                            new_audio_bytes = audio_bytes[new_bytes_start:]

                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                previous_chunk_tail,
                                new_audio_bytes,
                                sample_rate_hz=16000,
                                crossfade_ms=CROSSFADE_MS,
                            )

                            samples_emitted_in_this_chunk = len(to_emit) // 2
                            total_samples_emitted_to_user += samples_emitted_in_this_chunk

                        if to_emit:
                            total_audio_chunks += 1
                            yield to_emit

        # Final chunk processing
        if len(semantic_buffer) > last_decode_count:
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,
                global_ids=global_ids,  # Use captured globals
                use_sliding_window=False,
                trim_warmup=False
            )
            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    samples_emitted_in_final = len(to_emit) // 2
                    total_samples_emitted_to_user += samples_emitted_in_final

                    if to_emit:
                        total_audio_chunks += 1
                        yield to_emit

        # Flush remaining tail
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            yield previous_chunk_tail

        t_end = time.time()
        request_metrics.update(
            {
                "request_id": request_id,
                "speaker": speaker,
                "audio_duration_seconds": total_samples_emitted_to_user / 16000,
                "audio_chunks": total_audio_chunks,
                "semantic_tokens": len(semantic_buffer),
                "global_tokens": len(global_ids),
                "generation_ms": int((t_end - t_start) * 1000),
            }
        )
        logger.info("tts_stream_continuation_complete", extra=request_metrics)


Input:
*** Begin Patch
*** Update File: veena3modal/core/streaming_pipeline.py
@@
     async def generate_speech_stream_indic_first_chunk(
         self,
         speaker: str,
         text: str,
@@
         Thread Safety:
             This method is stateless and thread-safe. Each call creates its own
             request-scoped state. No global state is modified or shared.
         """
-        import time
-        t_start = time.time()
-        request_metrics = request_metrics or {}
-        request_metrics.setdefault("request_id", request_id)
-
-        # Build prompt using Indic prompt builder
-        prompt = self.prompt_builder.build_prefix(speaker, text)
-
-        # Configure sampling
-        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
-        sampling_params = SamplingParams(
-            temperature=temperature,
-            top_k=top_k,
-            top_p=top_p,
-            max_tokens=max_tokens,
-            min_tokens=DEFAULT_MIN_TOKENS,
-            repetition_penalty=repetition_penalty,
-            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
-            skip_special_tokens=False,
-            seed=seed,
-        )
-
-        # BiCodec token buffers
-        semantic_buffer = []
-        global_buffer = []
-        processed_token_count = 0  # Track how many vLLM tokens we've parsed
-        total_audio_chunks = 0
-
-        # BiCodec streaming configuration
-        EXPECTED_GLOBAL_COUNT = 32
-        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
-        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
-        CROSSFADE_MS = 50
-
-        # Generate unique request ID
-        import uuid
-        if request_id is None:
-            request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
-            request_metrics["request_id"] = request_id
-
-        # Reuse pre-warmed token parser (singleton, created once in __init__)
-        token_parser = self.token_parser
-        if token_parser is None:
-            raise RuntimeError("BiCodecTokenParser unavailable for first-chunk streaming")
-
-        results_generator = self.model.engine.generate(
-            prompt=prompt,
-            sampling_params=sampling_params,
-            request_id=request_id,
-        )
-
-        # Track state
-        last_decode_count = 0
-        total_samples_emitted_to_user = 0
-        previous_chunk_tail: Optional[bytes] = None
-        captured_globals: List[int] = []  # Will be populated once we have 32 globals
-
-        # Stream tokens with BiCodec extraction
-        async for request_output in results_generator:
-            generated_ids = request_output.outputs[0].token_ids
-
-            # OPTIMIZATION: Incremental parsing (O(1) per new token)
-            new_token_ids = generated_ids[processed_token_count:]
-            processed_token_count = len(generated_ids)
-            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
-
-            # Capture global tokens once we have all 32
-            if not captured_globals and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
-                captured_globals = global_buffer[:EXPECTED_GLOBAL_COUNT]
-
-            # Process if we have all global tokens
-            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
-                semantic_count = len(semantic_buffer)
-                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
-                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
-                        last_decode_count = semantic_count
-
-                        all_semantic = semantic_buffer
-                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]
-
-                        if self.snac_decoder.enable_batching:
-                            audio_bytes = await self.snac_decoder.decode_single_async(
-                                semantic_ids=all_semantic,
-                                global_ids=window_global,
-                                trim_warmup=False,
-                                use_sliding_window=False
-                            )
-                        else:
-                            audio_bytes = self.snac_decoder.decode_streaming(
-                                semantic_ids=all_semantic,
-                                global_ids=window_global,
-                                use_sliding_window=False,
-                                trim_warmup=False
-                            )
-
-                        if audio_bytes:
-                            total_samples_decoded = len(audio_bytes) // 2
-
-                            if total_samples_decoded > total_samples_emitted_to_user:
-                                new_bytes_start = total_samples_emitted_to_user * 2
-                                new_audio_bytes = audio_bytes[new_bytes_start:]
-
-                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
-                                    previous_chunk_tail,
-                                    new_audio_bytes,
-                                    sample_rate_hz=16000,
-                                    crossfade_ms=CROSSFADE_MS,
-                                )
-
-                                samples_emitted_in_this_chunk = len(to_emit) // 2
-                                total_samples_emitted_to_user += samples_emitted_in_this_chunk
-
-                                if to_emit:
-                                    total_audio_chunks += 1
-                                    # Yield tuple: (audio_bytes, global_ids)
-                                    yield (to_emit, captured_globals)
-
-        # Final chunk processing
-        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
-            audio_bytes = self.snac_decoder.decode_streaming(
-                semantic_ids=semantic_buffer,
-                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
-                use_sliding_window=False,
-                trim_warmup=False
-            )
-            if audio_bytes:
-                total_samples_decoded = len(audio_bytes) // 2
-
-                if total_samples_decoded > total_samples_emitted_to_user:
-                    new_bytes_start = total_samples_emitted_to_user * 2
-                    new_audio_bytes = audio_bytes[new_bytes_start:]
-
-                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
-                        previous_chunk_tail,
-                        new_audio_bytes,
-                        sample_rate_hz=16000,
-                        crossfade_ms=CROSSFADE_MS,
-                    )
-
-                    samples_emitted_in_final = len(to_emit) // 2
-                    total_samples_emitted_to_user += samples_emitted_in_final
-
-                        if to_emit:
-                            total_audio_chunks += 1
-                            yield (to_emit, captured_globals)
-
-        # Flush remaining tail
-        if previous_chunk_tail:
-            total_audio_chunks += 1
-            tail_samples = len(previous_chunk_tail) // 2
-            total_samples_emitted_to_user += tail_samples
-            yield (previous_chunk_tail, captured_globals)
-
-        t_end = time.time()
-        audio_duration_s = total_samples_emitted_to_user / 16000
-        request_metrics.update(
-            {
-                "request_id": request_id,
-                "speaker": speaker,
-                "audio_duration_seconds": audio_duration_s,
-                "audio_chunks": total_audio_chunks,
-                "text_chunked": True,
-                "captured_global_count": len(captured_globals),
-                "semantic_tokens": len(semantic_buffer),
-                "generation_ms": int((t_end - t_start) * 1000),
-            }
-        )
-        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)
+        import time
+        import torch
+
+        t_start = time.time()
+        emit_timing = _detailed_timing_enabled()
+        request_metrics = request_metrics or {}
+        request_metrics.update(
+            {
+                "request_id": request_id,
+                "speaker": speaker,
+                "text_length": len(text),
+                "text_chunked": True,
+            }
+        )
+
+        # Build prompt using Indic prompt builder
+        prompt = self.prompt_builder.build_prefix(speaker, text)
+
+        # Configure sampling
+        sampling_params = SamplingParams(
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            max_tokens=max_tokens,
+            min_tokens=DEFAULT_MIN_TOKENS,
+            repetition_penalty=repetition_penalty,
+            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
+            skip_special_tokens=False,
+            seed=seed,
+        )
+
+        # BiCodec token buffers
+        semantic_buffer = []
+        global_buffer = []
+        processed_token_count = 0  # Track how many vLLM tokens we've parsed
+        total_audio_chunks = 0
+
+        # BiCodec streaming configuration
+        EXPECTED_GLOBAL_COUNT = 32
+        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10
+        DECODE_INTERVAL = 48
+        CROSSFADE_MS = 50
+
+        # Generate unique request ID
+        import uuid
+        if request_id is None:
+            request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
+            request_metrics["request_id"] = request_id
+
+        # Reuse pre-warmed token parser (singleton, created once in __init__)
+        token_parser = self.token_parser
+        if token_parser is None:
+            raise RuntimeError("BiCodecTokenParser unavailable for first-chunk streaming")
+
+        results_generator = self.model.engine.generate(
+            prompt=prompt,
+            sampling_params=sampling_params,
+            request_id=request_id,
+        )
+
+        # Track state
+        batch_wall_ms: List[float] = []
+        batch_gpu_ms: List[float] = []
+        batch_token_counts: List[int] = []
+        parse_wall_ms_list: List[float] = []
+        last_decode_count = 0
+        total_samples_emitted_to_user = 0
+        previous_chunk_tail: Optional[bytes] = None
+        captured_globals: List[int] = []  # Will be populated once we have 32 globals
+        decode_calls = 0
+        decode_wall_ms = 0.0
+        decode_gpu_ms = 0.0
+        decode_wall_samples: List[float] = []
+        decode_gpu_samples: List[float] = []
+        t_first_token = None
+        t_first_chunk_ready = None
+        t_first_chunk_decoded = None
+        iteration_count = 0
+
+        # Stream tokens with BiCodec extraction
+        async for request_output in results_generator:
+            iteration_count += 1
+            t_iter = time.time()
+            if t_first_token is None:
+                t_first_token = t_iter
+
+            generated_ids = request_output.outputs[0].token_ids
+
+            parse_start = time.time()
+            # OPTIMIZATION: Incremental parsing (O(1) per new token)
+            new_token_ids = generated_ids[processed_token_count:]
+            processed_token_count = len(generated_ids)
+            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
+            parse_ms = (time.time() - parse_start) * 1000
+            parse_wall_ms_list.append(parse_ms)
+
+            batch_token_counts.append(len(new_token_ids))
+            batch_wall_ms_value = (time.time() - t_iter) * 1000
+            batch_wall_ms.append(batch_wall_ms_value)
+
+            if emit_timing:
+                logger.info(
+                    "tts_stream_llm_batch",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "batch_index": iteration_count,
+                        "batch_tokens": len(new_token_ids),
+                        "llm_batch_wall_ms": int(batch_wall_ms_value),
+                        "llm_batch_gpu_ms": int(batch_gpu_ms[-1]) if batch_gpu_ms else None,
+                        "text_chunked": True,
+                    },
+                )
+
+            # Capture global tokens once we have all 32
+            if not captured_globals and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
+                captured_globals = global_buffer[:EXPECTED_GLOBAL_COUNT]
+
+            # Process if we have all global tokens
+            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
+                if t_first_chunk_ready is None and len(semantic_buffer) >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
+                    t_first_chunk_ready = time.time()
+                semantic_count = len(semantic_buffer)
+                if semantic_count - last_decode_count >= DECODE_INTERVAL:
+                    last_decode_count = semantic_count
+
+                    all_semantic = semantic_buffer
+                    window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]
+                    decode_gpu_ms_this = None
+                    t_decode_start = time.time()
+                    if emit_timing and torch.cuda.is_available():
+                        torch.cuda.synchronize()
+                        decode_start_event = torch.cuda.Event(enable_timing=True)
+                        decode_end_event = torch.cuda.Event(enable_timing=True)
+                        decode_start_event.record()
+
+                    if self.snac_decoder.enable_batching:
+                        audio_bytes = await self.snac_decoder.decode_single_async(
+                            semantic_ids=all_semantic,
+                            global_ids=window_global,
+                            trim_warmup=False,
+                            use_sliding_window=False
+                        )
+                    else:
+                        audio_bytes = self.snac_decoder.decode_streaming(
+                            semantic_ids=all_semantic,
+                            global_ids=window_global,
+                            use_sliding_window=False,
+                            trim_warmup=False
+                        )
+
+                    t_decode_end = time.time()
+                    decode_calls += 1
+                    decode_ms = (t_decode_end - t_decode_start) * 1000
+                    decode_wall_ms += decode_ms
+                    decode_wall_samples.append(decode_ms)
+                    if emit_timing and torch.cuda.is_available():
+                        decode_end_event.record()
+                        torch.cuda.synchronize()
+                        decode_gpu_ms_this = decode_start_event.elapsed_time(decode_end_event)
+                        decode_gpu_ms += decode_gpu_ms_this
+                        decode_gpu_samples.append(decode_gpu_ms_this)
+                        batch_gpu_ms.append(decode_gpu_ms_this)
+
+                    to_emit = b""
+                    if audio_bytes:
+                        total_samples_decoded = len(audio_bytes) // 2
+
+                        if total_samples_decoded > total_samples_emitted_to_user:
+                            new_bytes_start = total_samples_emitted_to_user * 2
+                            new_audio_bytes = audio_bytes[new_bytes_start:]
+
+                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
+                                previous_chunk_tail,
+                                new_audio_bytes,
+                                sample_rate_hz=16000,
+                                crossfade_ms=CROSSFADE_MS,
+                            )
+
+                            samples_emitted_in_this_chunk = len(to_emit) // 2
+                            total_samples_emitted_to_user += samples_emitted_in_this_chunk
+
+                    if to_emit:
+                        total_audio_chunks += 1
+                        if t_first_chunk_decoded is None:
+                            t_first_chunk_decoded = t_decode_end
+                        if emit_timing:
+                            logger.info(
+                                "tts_stream_decode_batch",
+                                extra={
+                                    "speaker": speaker,
+                                    "request_id": request_id,
+                                    "batch_index": decode_calls,
+                                    "decode_wall_ms": int(decode_ms),
+                                    "decode_gpu_ms": int(decode_gpu_ms_this)
+                                    if decode_gpu_ms_this is not None else None,
+                                    "semantic_count": len(semantic_buffer),
+                                    "text_chunked": True,
+                                },
+                            )
+                        yield (to_emit, captured_globals)
+
+        # Final chunk processing
+        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
+            t_decode_start = time.time()
+            audio_bytes = self.snac_decoder.decode_streaming(
+                semantic_ids=semantic_buffer,
+                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
+                use_sliding_window=False,
+                trim_warmup=False
+            )
+            t_decode_end = time.time()
+            decode_ms = (t_decode_end - t_decode_start) * 1000
+            decode_calls += 1
+            decode_wall_ms += decode_ms
+            decode_wall_samples.append(decode_ms)
+
+            to_emit = b""
+            if audio_bytes:
+                total_samples_decoded = len(audio_bytes) // 2
+
+                if total_samples_decoded > total_samples_emitted_to_user:
+                    new_bytes_start = total_samples_emitted_to_user * 2
+                    new_audio_bytes = audio_bytes[new_bytes_start:]
+
+                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
+                        previous_chunk_tail,
+                        new_audio_bytes,
+                        sample_rate_hz=16000,
+                        crossfade_ms=CROSSFADE_MS,
+                    )
+
+                    samples_emitted_in_final = len(to_emit) // 2
+                    total_samples_emitted_to_user += samples_emitted_in_final
+
+            if to_emit:
+                total_audio_chunks += 1
+                if t_first_chunk_decoded is None:
+                    t_first_chunk_decoded = t_decode_end
+                if emit_timing:
+                    logger.info(
+                        "tts_stream_decode_batch",
+                        extra={
+                            "speaker": speaker,
+                            "request_id": request_id,
+                            "batch_index": decode_calls,
+                            "decode_wall_ms": int(decode_ms),
+                            "decode_gpu_ms": None,
+                            "semantic_count": len(semantic_buffer),
+                            "text_chunked": True,
+                        },
+                    )
+                yield (to_emit, captured_globals)
+
+        # Flush remaining tail
+        if previous_chunk_tail:
+            total_audio_chunks += 1
+            tail_samples = len(previous_chunk_tail) // 2
+            total_samples_emitted_to_user += tail_samples
+            yield (previous_chunk_tail, captured_globals)
+
+        t_end = time.time()
+        audio_duration_s = total_samples_emitted_to_user / 16000
+        ttfb_ms = (
+            (t_first_chunk_decoded - t_start) * 1000
+            if t_first_chunk_decoded is not None
+            else (t_first_chunk_ready - t_start) * 1000
+            if t_first_chunk_ready is not None
+            else (t_first_token - t_start) * 1000
+            if t_first_token is not None
+            else 0
+        )
+        request_metrics.update(
+            {
+                "request_id": request_id,
+                "speaker": speaker,
+                "generated_tokens": len(semantic_buffer) + len(global_buffer),
+                "semantic_tokens": len(semantic_buffer),
+                "global_tokens": len(captured_globals),
+                "llm_batch_count": len(batch_wall_ms),
+                "llm_batch_tokens": batch_token_counts,
+                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
+                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
+                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_parse_ms": int(sum(parse_wall_ms_list)),
+                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
+                "decode_calls": decode_calls,
+                "llm_decode_wall_ms_total": int(decode_wall_ms),
+                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
+                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "llm_decode_gpu_ms_p50": int(statistics.median(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "audio_duration_seconds": audio_duration_s,
+                "audio_chunks": total_audio_chunks,
+                "captured_global_count": len(captured_globals),
+                "text_chunked": True,
+                "ttfb_ms": int(ttfb_ms),
+                "generation_ms": int((t_end - t_start) * 1000),
+            }
+        )
+        if batch_token_counts:
+            request_metrics["tokens_per_batch_min"] = min(batch_token_counts)
+            request_metrics["tokens_per_batch_max"] = max(batch_token_counts)
+            request_metrics["tokens_per_batch_p50"] = int(statistics.median(batch_token_counts))
+
+        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


I fixed first-chunk streaming timing aggregation; I’ll mirror the same detailed decode/parse instrumentation in continuation mode and ensure completion metrics are consistent with first-chunk path.


Input:
*** Begin Patch
*** Update File: veena3modal/core/streaming_pipeline.py
@@
     async def generate_speech_stream_indic_continuation(
         self,
         speaker: str,
         text: str,
@@
         Thread Safety:
             Fully thread-safe. All state is request-scoped and passed explicitly.
         """
-        import time
-        t_start = time.time()
-        request_metrics = request_metrics or {}
-        request_metrics.setdefault("request_id", request_id)
-
-        # Validate global tokens
-        EXPECTED_GLOBAL_COUNT = 32
-        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
-            raise ValueError(
-                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
-                f"global_ids must be captured from first chunk generation."
-            )
-
-        # Build prompt WITH pre-filled global tokens
-        # This tells the model to skip global generation and go straight to semantic tokens
-        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)
-
-        # Configure sampling
-        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
-        sampling_params = SamplingParams(
-            temperature=temperature,
-            top_k=top_k,
-            top_p=top_p,
-            max_tokens=max_tokens,
-            min_tokens=DEFAULT_MIN_TOKENS,
-            repetition_penalty=repetition_penalty,
-            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
-            skip_special_tokens=False,
-            seed=seed,
-        )
-
-        # BiCodec token buffers
-        semantic_buffer = []
-        global_buffer_unused = []  # Not used in continuation, but needed for parser API
-        processed_token_count = 0  # Track how many vLLM tokens we've parsed
-        total_audio_chunks = 0
-
-        # BiCodec streaming configuration
-        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
-        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
-        CROSSFADE_MS = 50
-
-        # Generate unique request ID
-        import uuid
-        if request_id is None:
-            request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
-            request_metrics["request_id"] = request_id
-
-        # Reuse pre-warmed token parser (singleton, created once in __init__)
-        token_parser = self.token_parser
-        if token_parser is None:
-            raise RuntimeError("BiCodecTokenParser unavailable for continuation streaming")
-
-        results_generator = self.model.engine.generate(
-            prompt=prompt,
-            sampling_params=sampling_params,
-            request_id=request_id,
-        )
-
-        # Track state
-        last_decode_count = 0
-        total_samples_emitted_to_user = 0
-        previous_chunk_tail: Optional[bytes] = None
-
-        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
-        async for request_output in results_generator:
-            generated_ids = request_output.outputs[0].token_ids
-
-            # OPTIMIZATION: Incremental parsing (O(1) per new token)
-            # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
-            new_token_ids = generated_ids[processed_token_count:]
-            processed_token_count = len(generated_ids)
-            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)
-
-            semantic_count = len(semantic_buffer)
-            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
-                if semantic_count - last_decode_count >= DECODE_INTERVAL:
-                    last_decode_count = semantic_count
-
-                    # Use pre-captured global tokens for decoding
-                    if self.snac_decoder.enable_batching:
-                        audio_bytes = await self.snac_decoder.decode_single_async(
-                            semantic_ids=semantic_buffer,
-                            global_ids=global_ids,  # Use captured globals
-                            trim_warmup=False,
-                            use_sliding_window=False
-                        )
-                    else:
-                        audio_bytes = self.snac_decoder.decode_streaming(
-                            semantic_ids=semantic_buffer,
-                            global_ids=global_ids,  # Use captured globals
-                            use_sliding_window=False,
-                            trim_warmup=False
-                        )
-
-                    if audio_bytes:
-                        total_samples_decoded = len(audio_bytes) // 2
-
-                        if total_samples_decoded > total_samples_emitted_to_user:
-                            new_bytes_start = total_samples_emitted_to_user * 2
-                            new_audio_bytes = audio_bytes[new_bytes_start:]
-
-                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
-                                previous_chunk_tail,
-                                new_audio_bytes,
-                                sample_rate_hz=16000,
-                                crossfade_ms=CROSSFADE_MS,
-                            )
-
-                            samples_emitted_in_this_chunk = len(to_emit) // 2
-                            total_samples_emitted_to_user += samples_emitted_in_this_chunk
-
-                        if to_emit:
-                            total_audio_chunks += 1
-                            yield to_emit
-
-        # Final chunk processing
-        if len(semantic_buffer) > last_decode_count:
-            audio_bytes = self.snac_decoder.decode_streaming(
-                semantic_ids=semantic_buffer,
-                global_ids=global_ids,  # Use captured globals
-                use_sliding_window=False,
-                trim_warmup=False
-            )
-            if audio_bytes:
-                total_samples_decoded = len(audio_bytes) // 2
-
-                if total_samples_decoded > total_samples_emitted_to_user:
-                    new_bytes_start = total_samples_emitted_to_user * 2
-                    new_audio_bytes = audio_bytes[new_bytes_start:]
-
-                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
-                        previous_chunk_tail,
-                        new_audio_bytes,
-                        sample_rate_hz=16000,
-                        crossfade_ms=CROSSFADE_MS,
-                    )
-
-                    samples_emitted_in_final = len(to_emit) // 2
-                    total_samples_emitted_to_user += samples_emitted_in_final
-
-                    if to_emit:
-                        total_audio_chunks += 1
-                        yield to_emit
-
-        # Flush remaining tail
-        if previous_chunk_tail:
-            total_audio_chunks += 1
-            tail_samples = len(previous_chunk_tail) // 2
-            total_samples_emitted_to_user += tail_samples
-            yield previous_chunk_tail
-
-        t_end = time.time()
-        request_metrics.update(
-            {
-                "request_id": request_id,
-                "speaker": speaker,
-                "audio_duration_seconds": total_samples_emitted_to_user / 16000,
-                "audio_chunks": total_audio_chunks,
-                "semantic_tokens": len(semantic_buffer),
-                "global_tokens": len(global_ids),
-                "generation_ms": int((t_end - t_start) * 1000),
-            }
-        )
-        logger.info("tts_stream_continuation_complete", extra=request_metrics)
+        import time
+        import torch
+
+        t_start = time.time()
+        emit_timing = _detailed_timing_enabled()
+        request_metrics = request_metrics or {}
+        request_metrics.update(
+            {
+                "request_id": request_id,
+                "speaker": speaker,
+                "text_length": len(text),
+                "text_chunked": True,
+            }
+        )
+
+        # Validate global tokens
+        EXPECTED_GLOBAL_COUNT = 32
+        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
+            raise ValueError(
+                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
+                f"global_ids must be captured from first chunk generation."
+            )
+
+        # Build prompt WITH pre-filled global tokens
+        # This tells the model to skip global generation and go straight to semantic tokens
+        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)
+
+        # Configure sampling
+        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
+        sampling_params = SamplingParams(
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            max_tokens=max_tokens,
+            min_tokens=DEFAULT_MIN_TOKENS,
+            repetition_penalty=repetition_penalty,
+            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
+            skip_special_tokens=False,
+            seed=seed,
+        )
+
+        # BiCodec token buffers
+        semantic_buffer = []
+        global_buffer_unused = []  # Not used in continuation, but needed for parser API
+        processed_token_count = 0  # Track how many vLLM tokens we've parsed
+        total_audio_chunks = 0
+
+        # BiCodec streaming configuration
+        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
+        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
+        CROSSFADE_MS = 50
+
+        # Generate unique request ID
+        import uuid
+        if request_id is None:
+            request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
+            request_metrics["request_id"] = request_id
+
+        # Reuse pre-warmed token parser (singleton, created once in __init__)
+        token_parser = self.token_parser
+        if token_parser is None:
+            raise RuntimeError("BiCodecTokenParser unavailable for continuation streaming")
+
+        results_generator = self.model.engine.generate(
+            prompt=prompt,
+            sampling_params=sampling_params,
+            request_id=request_id,
+        )
+
+        # Track state
+        batch_wall_ms: List[float] = []
+        batch_gpu_ms: List[float] = []
+        batch_token_counts: List[int] = []
+        parse_wall_ms_list: List[float] = []
+        decode_calls = 0
+        decode_wall_ms = 0.0
+        decode_gpu_ms = 0.0
+        decode_wall_samples: List[float] = []
+        decode_gpu_samples: List[float] = []
+        t_first_token = None
+        t_first_chunk_ready = None
+        t_first_chunk_decoded = None
+        iteration_count = 0
+        last_decode_count = 0
+        total_samples_emitted_to_user = 0
+        previous_chunk_tail: Optional[bytes] = None
+
+        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
+        async for request_output in results_generator:
+            iteration_count += 1
+            t_iter = time.time()
+            if t_first_token is None:
+                t_first_token = t_iter
+
+            generated_ids = request_output.outputs[0].token_ids
+
+            # OPTIMIZATION: Incremental parsing (O(1) per new token)
+            # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
+            parse_start = time.time()
+            new_token_ids = generated_ids[processed_token_count:]
+            processed_token_count = len(generated_ids)
+            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)
+            parse_ms = (time.time() - parse_start) * 1000
+            parse_wall_ms_list.append(parse_ms)
+            batch_token_counts.append(len(new_token_ids))
+            batch_wall_ms_value = (time.time() - t_iter) * 1000
+            batch_wall_ms.append(batch_wall_ms_value)
+            if emit_timing:
+                logger.info(
+                    "tts_stream_llm_batch",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "batch_index": iteration_count,
+                        "batch_tokens": len(new_token_ids),
+                        "llm_batch_wall_ms": int(batch_wall_ms_value),
+                        "llm_batch_gpu_ms": int(batch_gpu_ms[-1]) if batch_gpu_ms else None,
+                        "text_chunked": True,
+                    },
+                )
+
+            semantic_count = len(semantic_buffer)
+            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
+                if t_first_chunk_ready is None:
+                    t_first_chunk_ready = time.time()
+                if semantic_count - last_decode_count >= DECODE_INTERVAL:
+                    last_decode_count = semantic_count
+
+                    decode_gpu_ms_this = None
+                    t_decode_start = time.time()
+                    if emit_timing and torch.cuda.is_available():
+                        torch.cuda.synchronize()
+                        decode_start_event = torch.cuda.Event(enable_timing=True)
+                        decode_end_event = torch.cuda.Event(enable_timing=True)
+                        decode_start_event.record()
+
+                    # Use pre-captured global tokens for decoding
+                    if self.snac_decoder.enable_batching:
+                        audio_bytes = await self.snac_decoder.decode_single_async(
+                            semantic_ids=semantic_buffer,
+                            global_ids=global_ids,  # Use captured globals
+                            trim_warmup=False,
+                            use_sliding_window=False
+                        )
+                    else:
+                        audio_bytes = self.snac_decoder.decode_streaming(
+                            semantic_ids=semantic_buffer,
+                            global_ids=global_ids,  # Use captured globals
+                            use_sliding_window=False,
+                            trim_warmup=False
+                        )
+
+                    t_decode_end = time.time()
+                    decode_calls += 1
+                    decode_ms = (t_decode_end - t_decode_start) * 1000
+                    decode_wall_ms += decode_ms
+                    decode_wall_samples.append(decode_ms)
+                    if emit_timing and torch.cuda.is_available():
+                        decode_end_event.record()
+                        torch.cuda.synchronize()
+                        decode_gpu_ms_this = decode_start_event.elapsed_time(decode_end_event)
+                        decode_gpu_ms += decode_gpu_ms_this
+                        decode_gpu_samples.append(decode_gpu_ms_this)
+                        batch_gpu_ms.append(decode_gpu_ms_this)
+
+                    to_emit = b""
+                    if audio_bytes:
+                        total_samples_decoded = len(audio_bytes) // 2
+
+                        if total_samples_decoded > total_samples_emitted_to_user:
+                            new_bytes_start = total_samples_emitted_to_user * 2
+                            new_audio_bytes = audio_bytes[new_bytes_start:]
+
+                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
+                                previous_chunk_tail,
+                                new_audio_bytes,
+                                sample_rate_hz=16000,
+                                crossfade_ms=CROSSFADE_MS,
+                            )
+
+                            samples_emitted_in_this_chunk = len(to_emit) // 2
+                            total_samples_emitted_to_user += samples_emitted_in_this_chunk
+
+                    if to_emit:
+                        total_audio_chunks += 1
+                        if t_first_chunk_decoded is None:
+                            t_first_chunk_decoded = t_decode_end
+                        if emit_timing:
+                            logger.info(
+                                "tts_stream_decode_batch",
+                                extra={
+                                    "speaker": speaker,
+                                    "request_id": request_id,
+                                    "batch_index": decode_calls,
+                                    "decode_wall_ms": int(decode_ms),
+                                    "decode_gpu_ms": int(decode_gpu_ms_this)
+                                    if decode_gpu_ms_this is not None else None,
+                                    "semantic_count": len(semantic_buffer),
+                                    "text_chunked": True,
+                                },
+                            )
+                        yield to_emit
+
+        # Final chunk processing
+        if len(semantic_buffer) > last_decode_count:
+            t_decode_start = time.time()
+            audio_bytes = self.snac_decoder.decode_streaming(
+                semantic_ids=semantic_buffer,
+                global_ids=global_ids,  # Use captured globals
+                use_sliding_window=False,
+                trim_warmup=False
+            )
+            t_decode_end = time.time()
+            decode_ms = (t_decode_end - t_decode_start) * 1000
+            decode_calls += 1
+            decode_wall_ms += decode_ms
+            decode_wall_samples.append(decode_ms)
+
+            to_emit = b""
+            if audio_bytes:
+                total_samples_decoded = len(audio_bytes) // 2
+
+                if total_samples_decoded > total_samples_emitted_to_user:
+                    new_bytes_start = total_samples_emitted_to_user * 2
+                    new_audio_bytes = audio_bytes[new_bytes_start:]
+
+                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
+                        previous_chunk_tail,
+                        new_audio_bytes,
+                        sample_rate_hz=16000,
+                        crossfade_ms=CROSSFADE_MS,
+                    )
+
+                    samples_emitted_in_final = len(to_emit) // 2
+                    total_samples_emitted_to_user += samples_emitted_in_final
+
+                    if to_emit:
+                        total_audio_chunks += 1
+                        if t_first_chunk_decoded is None:
+                            t_first_chunk_decoded = t_decode_end
+                        if emit_timing:
+                            logger.info(
+                                "tts_stream_decode_batch",
+                                extra={
+                                    "speaker": speaker,
+                                    "request_id": request_id,
+                                    "batch_index": decode_calls,
+                                    "decode_wall_ms": int(decode_ms),
+                                    "decode_gpu_ms": None,
+                                    "semantic_count": len(semantic_buffer),
+                                    "text_chunked": True,
+                                },
+                            )
+                        yield to_emit
+
+        # Flush remaining tail
+        if previous_chunk_tail:
+            total_audio_chunks += 1
+            tail_samples = len(previous_chunk_tail) // 2
+            total_samples_emitted_to_user += tail_samples
+            yield previous_chunk_tail
+
+        t_end = time.time()
+        ttfb_ms = (
+            (t_first_chunk_decoded - t_start) * 1000
+            if t_first_chunk_decoded is not None
+            else (t_first_chunk_ready - t_start) * 1000
+            if t_first_chunk_ready is not None
+            else (t_first_token - t_start) * 1000
+            if t_first_token is not None
+            else 0
+        )
+        request_metrics.update(
+            {
+                "request_id": request_id,
+                "speaker": speaker,
+                "generated_tokens": len(semantic_buffer) + len(global_ids),
+                "semantic_tokens": len(semantic_buffer),
+                "global_tokens": len(global_ids),
+                "llm_batch_count": len(batch_wall_ms),
+                "llm_batch_tokens": batch_token_counts,
+                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
+                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
+                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_parse_ms": int(sum(parse_wall_ms_list)),
+                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
+                "decode_calls": decode_calls,
+                "llm_decode_wall_ms_total": int(decode_wall_ms),
+                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
+                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "llm_decode_gpu_ms_p50": int(statistics.median(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "audio_duration_seconds": total_samples_emitted_to_user / 16000,
+                "audio_chunks": total_audio_chunks,
+                "text_chunked": True,
+                "ttfb_ms": int(ttfb_ms),
+                "generation_ms": int((t_end - t_start) * 1000),
+            }
+        )
+        if batch_token_counts:
+            request_metrics["tokens_per_batch_min"] = min(batch_token_counts)
+            request_metrics["tokens_per_batch_max"] = max(batch_token_counts)
+            request_metrics["tokens_per_batch_p50"] = int(statistics.median(batch_token_counts))
+        logger.info("tts_stream_continuation_complete", extra=request_metrics)
*** End Patch