Output:
Chunk ID: ab1226
Wall time: 0.0547 seconds
Process exited with code 0
Original token count: 1084
Output:
    active_requests = _with_inflight_delta(1)
    start_time = time.time()

    pipeline_timings = {}

    try:
        logger.info(
            "tts_runtime_nonstream_start",
            extra={
                "speaker": speaker,
                "request_id": request_id,
                "text_length": len(text),
                "request_inflight": active_requests,
                "temperature": temperature,
                "top_k": top_k,
                "top_p": top_p,
                "max_tokens": max_tokens,
            },
        )

        gen_kwargs = dict(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
        )

        if _timing_enabled():
            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
                **gen_kwargs,
                return_timing=True,
                request_id=request_id,
            )
        else:
            audio_bytes = await runtime.pipeline.generate_speech_indic(
                **gen_kwargs,
                request_id=request_id,
            )

        generation_time = time.time() - start_time
        metrics["generation_ms"] = int(generation_time * 1000)
        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time

        if pipeline_timings:
            metrics.update({
                "llm_token_total": pipeline_timings.get("token_total", 0),
                "llm_batch_count": pipeline_timings.get("batch_count", 0),
                "llm_batch_gpu_ms_total": pipeline_timings.get("llm_gpu_batch_ms_total", 0),
                "llm_batch_gpu_ms_min": pipeline_timings.get("llm_gpu_batch_ms_min", 0),
                "llm_batch_gpu_ms_max": pipeline_timings.get("llm_gpu_batch_ms_max", 0),
                "llm_batch_gpu_ms_p50": pipeline_timings.get("llm_gpu_batch_ms_p50", 0),
                "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
                "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
                "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
                "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
                "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
                "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
            })

        if audio_bytes:
            # Apply super-resolution if requested and available
            sample_rate = 16000
            logger.info(f"SR check: output_sample_rate={output_sample_rate!r}, "
                       f"sr_service={runtime.sr_service is not None}, "
                       f"is_loaded={runtime.sr_service.is_loaded if runtime.sr_service else False}")

            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
                try:
                    sr_start = time.time()
                    logger.info(f"Applying super-resolution to {len(audio_bytes)} bytes...")
                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
                    sr_time = (time.time() - sr_start) * 1000
                    metrics["sr_ms"] = int(sr_time)
                    metrics["sr_applied"] = True
                    sample_rate = 48000
                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
                except Exception as e:
                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
                    import traceback
                    traceback.print_exc()

            metrics["output_sample_rate"] = sample_rate

            # Calculate audio duration
                audio_duration = (len(audio_bytes) - 44) / (sample_rate * 2)  # -44 for WAV header
                metrics["audio_duration_seconds"] = max(0.0, audio_duration)

        if _timing_enabled():
            logger.info("tts_runtime_nonstream_complete", extra=metrics)


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


I spotted one concrete bug in `tts_runtime.generate_speech` while patching (over-indented audio-duration lines) and I’m correcting it while continuing to thread `request_id` through all runtime entrypoints.


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py
@@
 async def generate_speech_chunked(
     text: str,
     speaker: str,
     temperature: float = 0.8,
     top_k: int = 50,
@@
     sample_rate: int = 16000,
     output_sample_rate: str = "16khz",
+    request_id: Optional[str] = None,
 ) -> Tuple[Optional[bytes], Dict[str, Any]]:
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


<permissions instructions>
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `danger-full-access`: No filesystem sandboxing - all commands are permitted. Network access is enabled.
Approval policy is currently never. Do not provide the `sandbox_permissions` for any reason, commands will be rejected.
</permissions instructions>

<collaboration_mode># Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.
</collaboration_mode>

# AGENTS.md instructions for /home/ubuntu/veenaModal

<INSTRUCTIONS>
## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.
</INSTRUCTIONS>

<environment_context>
  <cwd>/home/ubuntu/veenaModal</cwd>
  <shell>bash</shell>
</environment_context>

The current TTS application here is an implementation in Modal labs. Which has autoscaling and other rules self scaling rules and everything suitable for my target audience. Now, i need you to create a local entry point or implementation point. then we will plan for optimising the pipeline, then we will discuss on stress testing, then we will discuss on scaling laws post optimisations locally. @veena3modal/app.py

the above part is performed by a different agent. and benchmarks are made. Now can you do the benchmarks on this newly optimized path to understand the concurrency and stress testing.

Now i need you to capture detailed time logs for each request happening in the GPU. like what happens with 1req, time per LLM token, time per batch decoding, and everything else. detailed time about where we are spending each minute in the GPU per generation.

and when you have the details for per request. you can obviously add more logging to get details of concurrency. so you understand what's the bottleneck for the peak concurrency. and then we will decide on effective methods in optimising this TTS engine.


Summary: auto


I’m going to finish the instrumentation path end-to-end (propagation of `request_id` and timing metrics across non-stream/chunked/streaming), then add a local benchmark script that drives concurrent requests and emits per-request GPU/latency timing so you can see batch/token bottlenecks clearly.


cd /home/ubuntu/veenaModal && git diff -- veena3modal/core/pipeline.py veena3modal/services/tts_runtime.py veena3modal/core/streaming_pipeline.py veena3modal/api/fastapi_app.py veena3modal/api/websocket_handler.py


Output:
Chunk ID: 6df301
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 14644
Output:
Total output lines: 1352

diff --git a/veena3modal/core/pipeline.py b/veena3modal/core/pipeline.py
index 9e44932..90ff4f7 100644
--- a/veena3modal/core/pipeline.py
+++ b/veena3modal/core/pipeline.py
@@ -7,8 +7,11 @@ Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).

 import asyncio
 import logging
+import os
 import re
+from statistics import mean, median
 from typing import Optional, List, Dict, Any, Tuple
+
 from vllm import SamplingParams
 import torch

@@ -25,13 +28,23 @@ from veena3modal.core.constants import (
 )

+def _detailed_timing_enabled() -> bool:
+    """
+    Return True when benchmark logs are enabled.
+
+    Controlled by environment variable:
+      VEENA3_TIMING_LOGS=true|1
+    """
+    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
+
+
 class SparkTTSPipeline:
     """
     End-to-end TTS pipeline for Spark TTS with BiCodec.

     Replaces SNAC-based token extraction with BiCodec regex-based extraction.
     """
-
+
     def __init__(
         self,
         model,
@@ -40,7 +53,7 @@ class SparkTTSPipeline:
     ):
         """
         Initialize pipeline.
-
+
         Args:
             model: SparkTTS Model instance
             prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
@@ -49,9 +62,9 @@ class SparkTTSPipeline:
         self.model = model
         self.prompt_builder = prompt_builder
         self.bicodec_decoder = bicodec_decoder
-
+
         print(f"🚀 SparkTTSPipeline initialized")
-
+
     async def generate_speech(
         self,
         speaker: str,
@@ -62,7 +75,9 @@ class SparkTTSPipeline:
         max_tokens: int = DEFAULT_MAX_TOKENS,
         repetition_penalty: float = 1.05,
         seed: Optional[int] = None,
-    ) -> Optional[bytes]:
+        return_timing: bool = False,
+        request_id: Optional[str] = None,
+    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
         """
         Generate speech audio (non-streaming) using Spark TTS.

@@ -77,53 +92,138 @@ class SparkTTSPipeline:
             top_p: Nucleus sampling (default: 1.0)
             max_tokens: Max BiCodec tokens to generate (default: 2048)
             seed: Random seed for reproducibility
-
+            return_timing: If true, returns (audio_bytes, metrics_dict)
+
         Returns:
-            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed
+            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
+            If return_timing is true, returns tuple(audio_bytes, metrics).
         """
         # Build prompt using Spark TTS format
         prompt = self.prompt_builder.build_prefix(speaker, text)
-
+
         # Configure sampling for Spark TTS
         sampling_params = SamplingParams(
             temperature=temperature,
             top_k=top_k,
             top_p=top_p,
             max_tokens=max_tokens,
-            repetition_penalty=repetition_penalty,  # Prevent token repetition (e.g., token 484 loop)
-            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>"
-            skip_special_tokens=False,  # Keep special tokens for BiCodec extraction
+            repetition_penalty=repetition_penalty,
+            stop=TRAINING_STOP_TOKEN_IDS,
+            skip_special_tokens=False,
             seed=seed,
         )
-
-        # Generate tokens
-        import uuid
-        request_id = f"req-{uuid.uuid4().hex[:12]}"
-
+
         # Use vLLM engine to generate
+        if request_id is None:
+            import uuid
+            request_id = f"req-{uuid.uuid4().hex[:12]}"
+
+        emit_timing = return_timing or _detailed_timing_enabled()
         results_generator = self.model.engine.generate(
             prompt=prompt,
             sampling_params=sampling_params,
             request_id=request_id,
         )
-
-        # Collect all outputs
+
+        # Collect all outputs and track generation timing
         final_output = None
-        async for request_output in results_generator:
+        token_batches: List[int] = []
+        batch_wall_ms: List[float] = []
+        batch_gpu_ms: List[float] = []
+        total_token_ids = 0
+        first_token_ms: Optional[float] = None
+        t_start = asyncio.get_event_loop().time()
+
+        iterator = results_generator.__aiter__()
+        while True:
+            wall_start = asyncio.get_event_loop().time()
+            start_event = None
+            end_event = None
+            if torch.cuda.is_available():
+                torch.cuda.synchronize()
+                start_event = torch.cuda.Event(enable_timing=True)
+                end_event = torch.cuda.Event(enable_timing=True)
+                start_event.record()
+
+            try:
+                request_output = await iterator.__anext__()
+            except StopAsyncIteration:
+                break
+
+            wall_end = asyncio.get_event_loop().time()
+            batch_ms = (wall_end - wall_start) * 1000
+            batch_wall_ms.append(batch_ms)
+
+            batch_gpu_ms_value: Optional[float] = None
+            if start_event is not None and end_event is not None:
+                end_event.record()
+                torch.cuda.synchronize()
+                batch_gpu_ms_value = start_event.elapsed_time(end_event)
+            if batch_gpu_ms_value is not None:
+                batch_gpu_ms.append(batch_gpu_ms_value)
+
+            total_generated = len(request_output.outputs[0].token_ids)
+            new_tokens = max(0, total_generated - total_token_ids)
+            total_token_ids = total_generated
+            token_batches.append(new_tokens)
+
+            if first_token_ms is None:
+                first_token_ms = (wall_end - t_start) * 1000
+
+            if emit_timing:
+                logger.info(
+                    "tts_llm_batch",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "batch_index": len(token_batches),
+                        "batch_tokens": new_tokens,
+                        "llm_batch_wall_ms": round(batch_ms, 3),
+                        "llm_batch_gpu_ms": round(batch_gpu_ms_value, 3)
+                        if batch_gpu_ms_value is not None
+                        else None,
+                    },
+                )
             final_output = request_output
-
+
         if final_output is None:
+            if emit_timing:
+                logger.warning(
+                    "tts_llm_no_output",
+                    extra={"speaker": speaker, "request_id": request_id},
+                )
             print(f"❌ Generation failed: no output")
+            if return_timing:
+                return None, {
+                    "request_id": request_id,
+                    "ttfb_ms": 0,
+                    "token_total": 0,
+                    "batch_count": 0,
+                }
             return None
-
+
         # Extract generated text (contains BiCodec tokens)
+        parse_start = asyncio.get_event_loop().time()
         generated_text = final_output.outputs[0].text
         generated_ids = final_output.outputs[0].token_ids
-
+
         # Extract BiCodec tokens using regex
         semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)
-
+
+        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000
+
         if not semantic_ids or not global_ids:
+            if emit_timing:
+                logger.error(
+                    "tts_bicodec_parse_failed",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "semantic_count": len(semantic_ids),
+                        "global_count": len(global_ids),
+                        "token_total": len(generated_ids),
+                    },
+                )
             logger.error(
                 "❌ No BiCodec tokens found in output",
                 extra={
@@ -136,8 +236,15 @@ class SparkTTSPipeline:
                     "generated_text_preview": generated_text[:500],
                 }
             )
+            if return_timing:
+                return None, {
+                    "request_id": request_id,
+                    "ttfb_ms": int(first_token_ms or 0),
+                    "token_total": len(generated_ids),
+                    "batch_count": len(token_batches),
+                }
             return None
-
+
         # Validate tokens
         if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
             logger.error("❌ BiCodec token validation failed", extra={
@@ -146,24 +253,71 @@ class SparkTTSPipeline:
                 "semantic_tokens": len(semantic_ids),
                 "global_tokens": len(global_ids),
             })
+            if return_timing:
+                return None, {
+                    "request_id": request_id,
+                    "ttfb_ms": int(first_token_ms or 0),
+                    "token_total": len(generated_ids),
+                    "batch_count": len(token_batches),
+                }
             return None
-
+
         # Decode to audio
+        decode_start = asyncio.get_event_loop().time()
         audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
-
+        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000
+
         if audio_bytes is None:
+            if emit_timing:
+                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
             logger.error("❌ BiCodec decode failed", extra={
                 "speaker": speaker,
                 "text_length": len(text),
             })
+            if return_timing:
+                return None, {
+                    "request_id": request_id,
+                    "ttfb_ms": int(first_token_ms or 0),
+                    "token_total": len(generated_ids),
+                    "batch_count": len(token_batches),
+                }
             return None
-
-        # Add WAV header
+
+        total_ms = (asyncio.get_event_loop().time() - t_start) * 1000
+
+        timing_metrics: Dict[str, Any] = {
+            "request_id": request_id,
+            "speaker": speaker,
+            "text_length": len(text),
+            "batch_count": len(token_batches),
+            "token_total": len(generated_ids),
+            "token_batches": token_batches,
+            "ttfb_ms": int(first_token_ms or 0),
+            "llm_wall_ms": int(sum(batch_wall_ms)),
+            "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
+            "llm_gpu_batch_ms_total": int(sum(batch_gpu_ms)),
+            "llm_gpu_batch_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
+            "llm_gpu_batch_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
+            "llm_gpu_batch_ms_p50": int(median(batch_gpu_ms)) if batch_gpu_ms else 0,
+            "decode_cpu_ms": int(decode_ms),
+            "parse_ms": int(parse_ms),
+            "total_ms": int(total_ms),
+        }
+        if token_batches:
+            timing_metrics["tokens_per_batch_min"] = min(token_batches)
+            timing_metrics["tokens_per_batch_max"] = max(token_batches)
+            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))
+
+        if emit_timing:
+            logger.info("tts_llm_complete", extra=timing_metrics)
+
         from veena3modal.audio.utils import add_wav_header
         wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)
-
+
+        if return_timing:
+            return wav_bytes, timing_metrics
         return wav_bytes
-
+
     async def generate_speech_indic(
         self,
         speaker: str,
@@ -174,7 +328,9 @@ class SparkTTSPipeline:
         max_tokens: int = DEFAULT_MAX_TOKENS,
         repetition_penalty: float = 1.05,
         seed: Optional[int] = None,
-    ) -> Optional[bytes]:
+        return_timing: bool = False,
+        request_id: Optional[str] = None,
+    ):
         """
         Generate speech audio for Spark TTS (speaker-based).

@@ -186,7 +342,6 @@ class SparkTTSPipeline:
             text: Text to synthesize with inline emotion tags
                 Examples:
                 - "Hello! Welcome."
-                - "[laughs] Hello there!"
                 - "Hello <laugh> this is fun!" (will be normalized)
                 - "नमस्ते! [excited] आज का दिन बहुत अच्छा है।"
             temperature: Sampling temperature
@@ -194,11 +349,13 @@ class SparkTTSPipeline:
             top_p: Nucleus sampling
             max_tokens: Max BiCodec tokens to generate
             seed: Random seed for reproducibility
+            return_timing: If true, returns (audio_bytes, metrics_dict)

         Returns:
-            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed
+            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
+            If return_timing is true, returns tuple(audio_bytes, metrics).
         """
-        # Delegate to generate_speech (same implementation for Spark TTS)
+        # Delegate to generate_speech() with the same implementation for Spark TTS
         return await self.generate_speech(
             speaker=speaker,
             text=text,
@@ -208,8 +365,10 @@ class SparkTTSPipeline:
             max_tokens=max_tokens,
             repetition_penalty=repetition_penalty,
             seed=seed,
+            return_timing=return_timing,
+            request_id=request_id,
         )
-
+
     def _extract_bicodec_tokens(self, generated_text: str) -> Tuple[List[int], List[int]]:
         """
         Extract BiCodec semantic and global tokens from generated text using regex.
@@ -217,10 +376,10 @@ class SparkTTSPipeline:
         Spark TTS generates tokens in the format:
         - <|bicodec_semantic_{id}|>
         - <|bicodec_global_{id}|>
-
+
         Args:
             generated_text: Generated text containing BiCodec token markers
-
+
         Returns:
             Tuple of (semantic_ids, global_ids)
             Returns ([], []) if no tokens found
@@ -228,14 +387,13 @@ class SparkTTSPipeline:
         # Extract semantic tokens
         semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", generated_text)
         semantic_ids = [int(t) for t in semantic_matches]
-
+
         # Extract global tokens
         global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
         global_ids = [int(t) for t in global_matches]
-
+
         # Only log if there's an issue
         if not semantic_ids and not global_ids:
             logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")
-
-        return semantic_ids, global_ids

+        return semantic_ids, global_ids
diff --git a/veena3modal/core/streaming_pipeline.py b/veena3modal/core/streaming_pipeline.py
index a6139fb..c8c9cf4 100644
--- a/veena3modal/core/streaming_pipeline.py
+++ b/veena3modal/core/streaming_pipeline.py
@@ -22,7 +22,10 @@ For SNAC (legacy):
 """

 import asyncio
+import os
 import re
+from statistics import median, mean
+import torch
 from typing import AsyncGenerator, Optional, List
 from vllm import SamplingParams
 from veena3modal.audio.crossfade import crossfade_bytes_int16
@@ -33,6 +36,7 @@ from veena3modal.core.constants import (
     CODE_START_TOKEN_ID,
     SNAC_MIN_ID,
     SNAC_MAX_ID,
+    TRAINING_STOP_TOKEN_IDS,
     DEFAULT_TEMPERATURE,
     DEFAULT_TOP_K,
     DEFAULT_TOP_P,
@@ -43,6 +47,11 @@ from veena3modal.core.constants import (
 )

+def _detailed_timing_enabled() -> bool:
+    """Enable per-batch token/decode timing logs."""
+    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
+
+
 class Veena3SlidingWindowPipeline:
     """
     Streaming TTS pipeline using sliding window approach.
@@ -73,6 +82,13 @@ class Veena3SlidingWindowPipeline:
         self.prompt_builder = prompt_builder
         self.snac_decoder = snac_decoder

+        # OPTIMIZATION: Pre-warm BiCodecTokenParser once at init, not per-request
+        # Saves ~123ms per streaming TTFB by avoiding repeated 166K-entry vocab iteration
+        tokenizer = getattr(model, "tokenizer", None)
+        if tokenizer is None:
+            tokenizer = getattr(model.engine, "tokenizer", None)
+        self.token_parser = BiCodecTokenParser(tokenizer) if tokenizer else None
+
         print(f"🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)")

     async def generate_speech_stream(
@@ -213,6 +229,7 @@ class Veena3SlidingWindowPipeline:
         max_tokens: int = DEFAULT_MAX_TOKENS,
         repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
         seed: Optional[int] = None,
+        request_id: Optional[str] = None,
     ) -> AsyncGenerator[bytes, None]:
         """
         Generate speech audio with sliding window streaming for Indic model.
@@ -237,19 +254,23 @@ class Veena3SlidingWindowPipeline:
         """
         import time
         t_start = time.time()
+        emit_timing = _detailed_timing_enabled()

         # Build prompt using Indic prompt builder
         prompt = self.prompt_builder.build_prefix(speaker, text)

         # Configure sampling (matching non-streaming pipeline parameters)
+        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID (128258)
+        # which doesn't exist in the Spark vocab and would cause generation to run to max_tokens
         sampling_params = SamplingParams(
             temperature=temperature,
-            top_k=top_k,  # Added for Spark TTS parity with non-streaming
+            top_k=top_k,
             top_p=top_p,
             max_tokens=max_tokens,
             min_tokens=DEFAULT_MIN_TOKENS,
             repetition_penalty=repetition_penalty,
-            stop_token_ids=[CODE_END_TOKEN_ID],  # Audio EOS
+            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
+            skip_special_tokens=False,  # Keep BiCodec tokens in output
             seed=seed,
         )

@@ -261,27 +282,32 @@ class Veena3SlidingWindowPipeline:

         # BiCodec streaming configuration
         EXPECTED_GLOBAL_COUNT = 32  # BiCodec uses 32 fixed global tokens (per paper)
-        MIN_SEMANTIC_FOR_FIRST_CHUNK = 16  # Start streaming after 16 semantic tokens (~320ms at 50 TPS)
-        DECODE_INTERVAL = 24  # Decode every 24 new semantic tokens (~480ms per chunk)
+        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8, -120ms TTFB)
+        # OPTIMIZATION: Increased from 24 to 48 to reduce O(n) re-decode calls by 2x.
+        # At 50 TPS, 48 tokens = ~960ms audio per chunk. Cuts total decode calls roughly in half.
+        DECODE_INTERVAL = 48
         CROSSFADE_MS = 50
+        # OPTIMIZATION: Windowed decode - only decode last WINDOW_SIZE tokens instead of ALL.
+        # BiCodec WaveGenerator has finite receptive field (~64 tokens based on kernel sizes).
+        # Window must be >= receptive_field + DECODE_INTERVAL to avoid boundary artifacts.
+        WINDOW_SIZE = 128  # 128 tokens covers receptive field + margin for crossfade

         # Generate tokens with vLLM (streaming)
-        # Generate unique request ID
-        import uuid
-        request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
-
-        # Get tokenizer from SparkTTSModel (falls back to engine tokenizer if unavailable)
-        tokenizer = getattr(self.model, "tokenizer", None)
-        if tokenizer is None:
-            tokenizer = getattr(self.model.engine, "tokenizer", None)
-        if tokenizer is None:
-…4644 tokens truncated…e to avoid async event loop conflicts
+        if num_engines <= 1:
+            _precompute_speaker_globals(_runtime)
+        else:
+            logger.info("Speaker globals cache skipped (multi-engine mode)")
+
         # Update FastAPI app with model version
         try:
             from veena3modal.api.fastapi_app import set_model_version
@@ -227,6 +272,96 @@ def initialize_runtime(
         raise RuntimeError(f"TTS runtime initialization failed: {e}") from e

+def _precompute_speaker_globals(runtime: TTSRuntime) -> None:
+    """
+    Pre-compute and cache global tokens for all 12 speakers at startup.
+
+    OPTIMIZATION: BiCodec streaming requires 32 "global tokens" before any audio can be emitted.
+    These encode speaker identity via FSQ quantization. Since we have only 12 fixed speakers,
+    we generate one short utterance per speaker at startup, capture the 32 global tokens,
+    and cache them. During streaming, we inject cached globals via build_prefix_with_globals(),
+    skipping the ~110ms global token pre-roll phase entirely.
+
+    This runs synchronously at startup (adds ~5-10s to cold start, saves ~110ms per streaming TTFB).
+    """
+    import asyncio
+    import re
+    from vllm import SamplingParams
+    from veena3modal.core.constants import (
+        SPEAKER_MAP, TRAINING_STOP_TOKEN_IDS,
+        DEFAULT_TEMPERATURE, DEFAULT_TOP_K, DEFAULT_TOP_P,
+    )
+
+    logger.info("Pre-computing global tokens for all speakers...")
+    t_start = time.time()
+
+    test_text = "Hello, this is a voice test."
+
+    sampling_params = SamplingParams(
+        temperature=DEFAULT_TEMPERATURE,
+        top_k=DEFAULT_TOP_K,
+        top_p=DEFAULT_TOP_P,
+        max_tokens=128,  # Only need ~60 tokens (32 global + some semantic)
+        stop=TRAINING_STOP_TOKEN_IDS,
+        skip_special_tokens=False,
+    )
+
+    async def _capture_globals_for_speaker(speaker_name: str) -> Optional[List[int]]:
+        """Generate a short utterance and capture the 32 global tokens."""
+        prompt = runtime.prompt_builder.build_prefix(speaker_name, test_text)
+        request_id = f"warmup-{speaker_name}-{int(time.time() * 1000)}"
+
+        final_output = None
+        async for request_output in runtime.model.engine.generate(
+            prompt=prompt,
+            sampling_params=sampling_params,
+            request_id=request_id,
+        ):
+            final_output = request_output
+
+        if final_output is None:
+            return None
+
+        generated_text = final_output.outputs[0].text
+        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
+        global_ids = [int(t) for t in global_matches]
+
+        if len(global_ids) >= 32:
+            return global_ids[:32]
+        return None
+
+    async def _run_all():
+        for speaker_name in SPEAKER_MAP.keys():
+            try:
+                global_ids = await _capture_globals_for_speaker(speaker_name)
+                if global_ids and len(global_ids) == 32:
+                    runtime.speaker_global_cache[speaker_name] = global_ids
+                    logger.info(f"  Cached globals for {speaker_name}")
+                else:
+                    logger.warning(f"  Failed to capture globals for {speaker_name}")
+            except Exception as e:
+                logger.warning(f"  Error caching globals for {speaker_name}: {e}")
+
+    # Run async warmup in event loop
+    try:
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            # Already in async context (unlikely at startup, but handle it)
+            import concurrent.futures
+            with concurrent.futures.ThreadPoolExecutor() as pool:
+                pool.submit(lambda: asyncio.run(_run_all())).result()
+        else:
+            loop.run_until_complete(_run_all())
+    except RuntimeError:
+        # No event loop exists yet
+        asyncio.run(_run_all())
+
+    elapsed = (time.time() - t_start) * 1000
+    cached = len(runtime.speaker_global_cache)
+    total = len(SPEAKER_MAP)
+    logger.info(f"Speaker globals cached: {cached}/{total} speakers in {elapsed:.0f}ms")
+
+
 async def generate_speech(
     text: str,
     speaker: str,
@@ -237,6 +372,7 @@ async def generate_speech(
     repetition_penalty: float = 1.05,
     seed: Optional[int] = None,
     output_sample_rate: str = "16khz",
+    request_id: Optional[str] = None,
 ) -> Tuple[Optional[bytes], Dict[str, Any]]:
     """
     Generate speech audio (non-streaming).
@@ -265,6 +401,7 @@ async def generate_speech(

     runtime = get_runtime()
     metrics = {
+        "request_id": request_id,
         "ttfb_ms": 0,
         "generation_ms": 0,
         "tokens_generated": 0,
@@ -272,12 +409,33 @@ async def generate_speech(
         "sr_applied": False,
         "output_sample_rate": 16000,
     }
+
+    if request_id is None:
+        import uuid
+        request_id = f"runtime-ns-{uuid.uuid4().hex[:10]}"
+        metrics["request_id"] = request_id

+    active_requests = _with_inflight_delta(1)
     start_time = time.time()
+
+    pipeline_timings = {}

     try:
-        # Generate audio using pipeline (always at 16kHz)
-        audio_bytes = await runtime.pipeline.generate_speech_indic(
+        logger.info(
+            "tts_runtime_nonstream_start",
+            extra={
+                "speaker": speaker,
+                "request_id": request_id,
+                "text_length": len(text),
+                "request_inflight": active_requests,
+                "temperature": temperature,
+                "top_k": top_k,
+                "top_p": top_p,
+                "max_tokens": max_tokens,
+            },
+        )
+
+        gen_kwargs = dict(
             speaker=speaker,
             text=text,
             temperature=temperature,
@@ -288,10 +446,39 @@ async def generate_speech(
             seed=seed,
         )

+        if _timing_enabled():
+            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
+                **gen_kwargs,
+                return_timing=True,
+                request_id=request_id,
+            )
+        else:
+            audio_bytes = await runtime.pipeline.generate_speech_indic(
+                **gen_kwargs,
+                request_id=request_id,
+            )
+
         generation_time = time.time() - start_time
         metrics["generation_ms"] = int(generation_time * 1000)
         metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time

+        if pipeline_timings:
+            metrics.update({
+                "llm_token_total": pipeline_timings.get("token_total", 0),
+                "llm_batch_count": pipeline_timings.get("batch_count", 0),
+                "llm_batch_gpu_ms_total": pipeline_timings.get("llm_gpu_batch_ms_total", 0),
+                "llm_batch_gpu_ms_min": pipeline_timings.get("llm_gpu_batch_ms_min", 0),
+                "llm_batch_gpu_ms_max": pipeline_timings.get("llm_gpu_batch_ms_max", 0),
+                "llm_batch_gpu_ms_p50": pipeline_timings.get("llm_gpu_batch_ms_p50", 0),
+                "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
+                "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
+                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
+                "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
+                "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
+                "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
+                "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
+            })
+
         if audio_bytes:
             # Apply super-resolution if requested and available
             sample_rate = 16000
@@ -317,14 +504,24 @@ async def generate_speech(
             metrics["output_sample_rate"] = sample_rate

             # Calculate audio duration
-            audio_duration = (len(audio_bytes) - 44) / (sample_rate * 2)  # -44 for WAV header
-            metrics["audio_duration_seconds"] = max(0.0, audio_duration)
+                audio_duration = (len(audio_bytes) - 44) / (sample_rate * 2)  # -44 for WAV header
+                metrics["audio_duration_seconds"] = max(0.0, audio_duration)
+
+        if _timing_enabled():
+            logger.info("tts_runtime_nonstream_complete", extra=metrics)

         return audio_bytes, metrics

     except Exception as e:
         logger.error(f"Speech generation failed: {e}")
         raise
+        finally:
+            active_requests = _with_inflight_delta(-1)
+            if _timing_enabled():
+                logger.info(
+                    "tts_runtime_nonstream_inflight",
+                    extra={"active_requests": active_requests},
+                )

 def _apply_super_resolution(audio_bytes: bytes, sr_service) -> bytes:
@@ -404,6 +601,7 @@ async def generate_speech_chunked(
     seed: Optional[int] = None,
     sample_rate: int = 16000,
     output_sample_rate: str = "16khz",
+    request_id: Optional[str] = None,
 ) -> Tuple[Optional[bytes], Dict[str, Any]]:
     """
     Generate speech with automatic text chunking for long inputs.
@@ -434,9 +632,20 @@ async def generate_speech_chunked(
         "output_sample_rate": 16000,
     }

+    active_requests = _with_inflight_delta(1)
     start_time = time.time()
-
+
     try:
+        if _timing_enabled():
+            logger.info(
+                "tts_runtime_chunking_start",
+                extra={
+                    "speaker": speaker,
+                    "text_length": len(text),
+                    "request_inflight": active_requests,
+                    "chunking": True,
+                },
+            )
         # Import long text processor
         from veena3modal.processing.long_text_processor import LongTextProcessor

@@ -495,11 +704,16 @@ async def generate_speech_chunked(
             audio_duration = (len(audio_bytes) - 44) / (final_sample_rate * 2)
             metrics["audio_duration_seconds"] = max(0.0, audio_duration)

+        if _timing_enabled():
+            metrics["active_requests"] = active_requests - 1
+            logger.info("tts_runtime_chunking_complete", extra=metrics)
         return audio_bytes, metrics

     except Exception as e:
         logger.error(f"Chunked speech generation failed: {e}")
         raise
+    finally:
+        _with_inflight_delta(-1)

 from typing import AsyncGenerator
@@ -515,6 +729,7 @@ async def generate_speech_streaming(
     repetition_penalty: float = 1.05,
     seed: Optional[int] = None,
     enable_chunking: bool = True,
+    request_id: Optional[str] = None,
 ) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
     """
     Generate speech audio with true streaming (yields chunks as they're generated).
@@ -567,58 +782,114 @@ async def generate_speech_streaming(
     first_chunk_time = None
     total_pcm_bytes = 0
     sample_rate = 16000  # BiCodec sample rate
-
-    # Check if we need text chunking
-    from veena3modal.processing.long_text_processor import LongTextProcessor
-    long_processor = LongTextProcessor(pipeline=runtime.pipeline)
-    needs_chunking = enable_chunking and long_processor.should_chunk(text)
-
-    if needs_chunking:
-        # Chunked streaming with voice consistency (global token caching)
-        metrics["text_chunked"] = True
-        async for audio_chunk, chunk_metrics in _stream_chunked_text(
-            runtime, long_processor, text, speaker, temperature, top_k, top_p,
-            max_tokens, repetition_penalty, seed, sample_rate
-        ):
-            # First chunk: prepend WAV header
-            if first_chunk_time is None:
-                first_chunk_time = time.time()
-                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
-                # Create streaming WAV header (size=0 for unknown length)
-                wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
-                audio_chunk = wav_header + audio_chunk
-
-            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
-            metrics["chunks_sent"] += 1
-            metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
-            metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
-
-            yield audio_chunk, metrics
-    else:
-        # Simple streaming (no text chunking)
-        async for audio_chunk in runtime.streaming_pipeline.generate_speech_stream_indic(
-            speaker=speaker,
-            text=text,
-            temperature=temperature,
-            top_k=top_k,
-            top_p=top_p,
-            max_tokens=max_tokens,
-            repetition_penalty=repetition_penalty,
-            seed=seed,
-        ):
-            # First chunk: prepend WAV header
-            if first_chunk_time is None:
-                first_chunk_time = time.time()
-                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
-                wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
-                audio_chunk = wav_header + audio_chunk
-
-            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
-            metrics["chunks_sent"] += 1
-            metrics["total_bytes"] = total_pcm_bytes + 44
-            metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
-
-            yield audio_chunk, metrics
+    active_requests = _with_inflight_delta(1)
+    if _timing_enabled():
+        logger.info(
+            "tts_runtime_stream_start",
+            extra={
+                "speaker": speaker,
+                "text_length": len(text),
+                "request_inflight": active_requests,
+                "chunking": enable_chunking,
+                "request_id": request_id,
+            },
+        )
+
+    try:
+        # Check if we need text chunking
+        from veena3modal.processing.long_text_processor import LongTextProcessor
+        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
+        needs_chunking = enable_chunking and long_processor.should_chunk(text)
+
+        if needs_chunking:
+            # Chunked streaming with voice consistency (global token caching)
+            metrics["text_chunked"] = True
+            async for audio_chunk, chunk_metrics in _stream_chunked_text(
+                runtime,
+                long_processor,
+                text,
+                speaker,
+                temperature,
+                top_k,
+                top_p,
+                max_tokens,
+                repetition_penalty,
+                seed,
+                sample_rate,
+                request_id,
+            ):
+                if first_chunk_time is None:
+                    first_chunk_time = time.time()
+                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
+                    # Create streaming WAV header (size=0 for unknown length)
+                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
+                    audio_chunk = wav_header + audio_chunk
+
+                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
+                metrics["chunks_sent"] += 1
+                metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
+                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
+                if chunk_metrics:
+                    metrics.update(chunk_metrics)
+
+                yield audio_chunk, metrics
+        else:
+            # Simple streaming (no text chunking)
+            # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
+            # use continuation mode to skip the ~110ms global token pre-roll.
+            # The model jumps straight to semantic token generation.
+            cached_globals = runtime.speaker_global_cache.get(speaker)
+
+            if cached_globals:
+                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
+                    speaker=speaker,
+                    text=text,
+                    global_ids=cached_globals,
+                    temperature=temperature,
+                    top_k=top_k,
+                    top_p=top_p,
+                    max_tokens=max_tokens,
+                    repetition_penalty=repetition_penalty,
+                    seed=seed,
+                    request_id=request_id,
+                )
+            else:
+                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
+                    speaker=speaker,
+                    text=text,
+                    temperature=temperature,
+                    top_k=top_k,
+                    top_p=top_p,
+                    max_tokens=max_tokens,
+                    repetition_penalty=repetition_penalty,
+                    seed=seed,
+                    request_id=request_id,
+                )
+
+            async for audio_chunk in stream_gen:
+                if first_chunk_time is None:
+                    first_chunk_time = time.time()
+                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
+                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
+                    audio_chunk = wav_header + audio_chunk
+
+                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
+                metrics["chunks_sent"] += 1
+                metrics["total_bytes"] = total_pcm_bytes + 44
+                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
+                yield audio_chunk, metrics
+    finally:
+        active_requests = _with_inflight_delta(-1)
+        if _timing_enabled():
+            logger.info(
+                "tts_runtime_stream_end",
+                extra={
+                    "speaker": speaker,
+                    "request_id": request_id,
+                    "request_inflight": active_requests,
+                    "chunks_sent": metrics.get("chunks_sent", 0),
+                },
+            )

 async def _stream_chunked_text(
@@ -633,6 +904,7 @@ async def _stream_chunked_text(
     repetition_penalty: float,
     seed: Optional[int],
     sample_rate: int,
+    request_id: Optional[str] = None,
 ) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
     """
     Internal helper: stream chunked text with voice consistency.
@@ -665,6 +937,7 @@ async def _stream_chunked_text(
                 max_tokens=max_tokens,
                 repetition_penalty=repetition_penalty,
                 seed=seed,
+                request_id=request_id,
             ):
                 if captured_globals is None and global_ids:
                     captured_globals = global_ids
@@ -692,6 +965,7 @@ async def _stream_chunked_text(
                     max_tokens=max_tokens,
                     repetition_penalty=repetition_penalty,
                     seed=seed,
+                    request_id=request_id,
                 ):
                     to_emit, previous_chunk_tail = crossfade_bytes_int16(
                         previous_chunk_tail,
@@ -713,6 +987,7 @@ async def _stream_chunked_text(
                     max_tokens=max_tokens,
                     repetition_penalty=repetition_penalty,
                     seed=seed,
+                    request_id=request_id,
                 ):
                     to_emit, previous_chunk_tail = crossfade_bytes_int16(
                         previous_chunk_tail,


cd /home/ubuntu/veenaModal && sed -n '1,260p' veena3modal/services/tts_runtime.py


cd /home/ubuntu/veenaModal && sed -n '260,560p' veena3modal/services/tts_runtime.py


cd /home/ubuntu/veenaModal && sed -n '1,320p' veena3modal/core/pipeline.py


cd /home/ubuntu/veenaModal && sed -n '1,280p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: a5890d
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 2408
Output:
"""
TTS runtime wrapper (container-scoped singleton).

In Modal, each container should load the model once and then handle many requests.
This wrapper owns:
- SparkTTSModel (vLLM engine)
- IndicPromptBuilder
- BiCodecDecoder
- SparkTTSPipeline + Veena3SlidingWindowPipeline
- optional SuperResolutionService

Imports framework-agnostic inference + processing code from `veena3modal/core`,
`veena3modal/processing`, and `veena3modal/audio`.
"""

from __future__ import annotations

import os
import sys
import time
import logging
from threading import Lock
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

logger = logging.getLogger(__name__)

_PROFILE_REQUEST_TIMING = "VEENA3_TIMING_LOGS"
_inflight_counter = 0
_inflight_lock = Lock()

def _timing_enabled() -> bool:
    """Return True when detailed request timing should be emitted."""
    return os.getenv(_PROFILE_REQUEST_TIMING, "").lower() in {"1", "true", "yes", "on"}

def _with_inflight_delta(delta: int) -> int:
    """Update in-flight request counter and return the new value."""
    global _inflight_counter
    with _inflight_lock:
        _inflight_counter = max(0, _inflight_counter + delta)
        return _inflight_counter

@dataclass
class TTSRuntime:
    """
    Holds long-lived, per-container inference objects.

    Thread Safety:
        This class is designed for async concurrency within a single container.
        The vLLM engine handles internal batching and scheduling.
        Do NOT share across processes without proper synchronization.
    """
    model: Any = None
    pipeline: Any = None
    streaming_pipeline: Any = None
    prompt_builder: Any = None
    bicodec_decoder: Any = None
    sr_service: Optional[Any] = None

    model_version: str = "not_loaded"
    is_loaded: bool = False
    load_time_ms: float = 0.0

    # Configuration
    model_path: str = ""
    bicodec_path: str = ""
    sr_checkpoint_dir: Optional[str] = None
    device: str = "cuda"

    # OPTIMIZATION: Pre-computed global tokens per speaker (eliminates ~110ms pre-roll in streaming)
    # Maps speaker_name -> list of 32 global token IDs captured at startup
    speaker_global_cache: Dict[str, List[int]] = field(default_factory=dict)

# Module-level singleton (per-container)
_runtime: Optional[TTSRuntime] = None

def get_runtime() -> Optional[TTSRuntime]:
    """Get the current TTS runtime singleton."""
    return _runtime

def is_initialized() -> bool:
    """Check if the runtime is initialized and ready."""
    return _runtime is not None and _runtime.is_loaded

def initialize_runtime(
    model_path: Optional[str] = None,
    bicodec_path: Optional[str] = None,
    sr_checkpoint_dir: Optional[str] = None,
    device: str = "cuda",
    hf_token: Optional[str] = None,
    gpu_memory_utilization: float = 0.25,
    enable_sr: bool = False,
    num_engines: int = 1,
) -> TTSRuntime:
    """
    Initialize the TTS runtime with all components.

    This should be called once per container (e.g., in Modal's @modal.enter).

    Args:
        model_path: Path to Spark TTS model (env: SPARK_TTS_MODEL_PATH)
        bicodec_path: Path to BiCodec model (env: BICODEC_MODEL_PATH, defaults to model_path)
        sr_checkpoint_dir: Path to super-resolution checkpoints (env: AP_BWE_CHECKPOINT_DIR)
        device: Device for inference (cuda/cpu)
        hf_token: HuggingFace token for private models (env: HF_TOKEN)
        gpu_memory_utilization: vLLM GPU memory fraction (default: 0.25)
        enable_sr: Enable super-resolution service
        num_engines: Number of vLLM engine instances (default: 1, set to 2-3 for Tier 3 optimization)

    Returns:
        Initialized TTSRuntime instance

    Raises:
        RuntimeError: If initialization fails
    """
    global _runtime

    start_time = time.time()

    # Resolve paths from env vars if not provided
    model_path = model_path or os.environ.get(
        'SPARK_TTS_MODEL_PATH',
        os.environ.get('MODEL_PATH', '/models/spark_tts_4speaker')
    )
    bicodec_path = bicodec_path or os.environ.get(
        'BICODEC_MODEL_PATH',
        model_path  # BiCodec is usually in the same directory
    )
    sr_checkpoint_dir = sr_checkpoint_dir or os.environ.get('AP_BWE_CHECKPOINT_DIR')
    hf_token = hf_token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')

    logger.info(f"Initializing TTS runtime...")
    logger.info(f"  Model path: {model_path}")
    logger.info(f"  BiCodec path: {bicodec_path}")
    logger.info(f"  SR enabled: {enable_sr}, path: {sr_checkpoint_dir}")

    try:
        # Import inference components (vendored into veena3modal for Modal deployment)
        from veena3modal.core.model_loader import SparkTTSModel
        from veena3modal.core.pipeline import SparkTTSPipeline
        from veena3modal.core.bicodec_decoder import BiCodecDecoder
        from veena3modal.processing.prompt_builder import IndicPromptBuilder

        # Load SparkTTS model with vLLM engine
        if num_engines > 1:
            # TIER 3 OPTIMIZATION: Multiple vLLM engines on same GPU
            # Each engine gets gpu_memory_utilization / num_engines to share GPU fairly.
            # E.g., 3 engines at 0.08 each = 0.24 total, ~8GB per engine, ~24GB total.
            from veena3modal.core.multi_engine import create_multi_engine_model
            per_engine_mem = gpu_memory_utilization / num_engines
            logger.info(f"Loading {num_engines} vLLM engines ({per_engine_mem:.2f} GPU mem each)...")
            model = create_multi_engine_model(
                model_path=model_path,
                num_engines=num_engines,
                hf_token=hf_token,
                gpu_memory_per_engine=per_engine_mem,
            )
        else:
            logger.info("Loading SparkTTS model with vLLM...")
            model = SparkTTSModel(
                model_path=model_path,
                hf_token=hf_token,
                gpu_memory_utilization=gpu_memory_utilization,
            )

        # Initialize prompt builder
        logger.info("Initializing prompt builder...")
        prompt_builder = IndicPromptBuilder(
            tokenizer=model.tokenizer,
            model=model,
        )

        # Initialize BiCodec decoder
        logger.info("Initializing BiCodec decoder...")
        bicodec_decoder = BiCodecDecoder(
            device=device,
            model_path=bicodec_path,
        )

        # Initialize pipeline
        logger.info("Initializing TTS pipeline...")
        pipeline = SparkTTSPipeline(
            model=model,
            prompt_builder=prompt_builder,
            bicodec_decoder=bicodec_decoder,
        )

        # Initialize streaming pipeline (for M4)
        streaming_pipeline = None
        try:
            from veena3modal.core.streaming_pipeline import Veena3SlidingWindowPipeline
            logger.info("Initializing streaming pipeline...")
            # NOTE: Parameter is named 'snac_decoder' for legacy reasons, but works with BiCodecDecoder
            streaming_pipeline = Veena3SlidingWindowPipeline(
                model=model,
                prompt_builder=prompt_builder,
                snac_decoder=bicodec_decoder,  # BiCodecDecoder is interface-compatible
            )
        except ImportError as e:
            logger.warning(f"Streaming pipeline not available: {e}")

        # Initialize super-resolution (optional)
        sr_service = None
        if enable_sr and sr_checkpoint_dir:
            try:
                from veena3modal.core.super_resolution import SuperResolutionService
                logger.info(f"Initializing super-resolution from {sr_checkpoint_dir}...")
                sr_service = SuperResolutionService(checkpoint_dir=sr_checkpoint_dir)
                # Load the model explicitly
                if sr_service.load_model(device=device):
                    logger.info("✅ Super-resolution model loaded successfully")
                else:
                    logger.warning("Super-resolution model failed to load")
                    sr_service = None
            except Exception as e:
                logger.warning(f"Super-resolution not available: {e}")

        # Determine model version
        model_version = os.path.basename(model_path.rstrip('/'))
        if not model_version:
            model_version = "spark-tts"

        load_time = (time.time() - start_time) * 1000

        # Create runtime
        _runtime = TTSRuntime(
            model=model,
            pipeline=pipeline,
            streaming_pipeline=streaming_pipeline,
            prompt_builder=prompt_builder,
            bicodec_decoder=bicodec_decoder,
            sr_service=sr_service,
            model_version=model_version,
            is_loaded=True,
            load_time_ms=load_time,
            model_path=model_path,
            bicodec_path=bicodec_path,
            sr_checkpoint_dir=sr_checkpoint_dir,
            device=device,
        )

        logger.info(f"✅ TTS runtime initialized in {load_time:.0f}ms")
        logger.info(f"   Model version: {model_version}")

        # OPTIMIZATION: Pre-compute global tokens for all speakers
        # Disabled with multi-engine to avoid async event loop conflicts
        if num_engines <= 1:
            _precompute_speaker_globals(_runtime)
        else:
            logger.info("Speaker globals cache skipped (multi-engine mode)")

        # Update FastAPI app with model version
        try:


Output:
Chunk ID: 3c1a2b
Wall time: 0.0505 seconds
Process exited with code 0
Original token count: 2905
Output:
        try:
            from veena3modal.api.fastapi_app import set_model_version
            set_model_version(model_version)
        except ImportError:
            pass

        return _runtime

    except Exception as e:
        logger.error(f"❌ Failed to initialize TTS runtime: {e}")
        import traceback
        traceback.print_exc()
        raise RuntimeError(f"TTS runtime initialization failed: {e}") from e

def _precompute_speaker_globals(runtime: TTSRuntime) -> None:
    """
    Pre-compute and cache global tokens for all 12 speakers at startup.

    OPTIMIZATION: BiCodec streaming requires 32 "global tokens" before any audio can be emitted.
    These encode speaker identity via FSQ quantization. Since we have only 12 fixed speakers,
    we generate one short utterance per speaker at startup, capture the 32 global tokens,
    and cache them. During streaming, we inject cached globals via build_prefix_with_globals(),
    skipping the ~110ms global token pre-roll phase entirely.

    This runs synchronously at startup (adds ~5-10s to cold start, saves ~110ms per streaming TTFB).
    """
    import asyncio
    import re
    from vllm import SamplingParams
    from veena3modal.core.constants import (
        SPEAKER_MAP, TRAINING_STOP_TOKEN_IDS,
        DEFAULT_TEMPERATURE, DEFAULT_TOP_K, DEFAULT_TOP_P,
    )

    logger.info("Pre-computing global tokens for all speakers...")
    t_start = time.time()

    test_text = "Hello, this is a voice test."

    sampling_params = SamplingParams(
        temperature=DEFAULT_TEMPERATURE,
        top_k=DEFAULT_TOP_K,
        top_p=DEFAULT_TOP_P,
        max_tokens=128,  # Only need ~60 tokens (32 global + some semantic)
        stop=TRAINING_STOP_TOKEN_IDS,
        skip_special_tokens=False,
    )

    async def _capture_globals_for_speaker(speaker_name: str) -> Optional[List[int]]:
        """Generate a short utterance and capture the 32 global tokens."""
        prompt = runtime.prompt_builder.build_prefix(speaker_name, test_text)
        request_id = f"warmup-{speaker_name}-{int(time.time() * 1000)}"

        final_output = None
        async for request_output in runtime.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        ):
            final_output = request_output

        if final_output is None:
            return None

        generated_text = final_output.outputs[0].text
        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
        global_ids = [int(t) for t in global_matches]

        if len(global_ids) >= 32:
            return global_ids[:32]
        return None

    async def _run_all():
        for speaker_name in SPEAKER_MAP.keys():
            try:
                global_ids = await _capture_globals_for_speaker(speaker_name)
                if global_ids and len(global_ids) == 32:
                    runtime.speaker_global_cache[speaker_name] = global_ids
                    logger.info(f"  Cached globals for {speaker_name}")
                else:
                    logger.warning(f"  Failed to capture globals for {speaker_name}")
            except Exception as e:
                logger.warning(f"  Error caching globals for {speaker_name}: {e}")

    # Run async warmup in event loop
    try:
        loop = asyncio.get_event_loop()
        if loop.is_running():
            # Already in async context (unlikely at startup, but handle it)
            import concurrent.futures
            with concurrent.futures.ThreadPoolExecutor() as pool:
                pool.submit(lambda: asyncio.run(_run_all())).result()
        else:
            loop.run_until_complete(_run_all())
    except RuntimeError:
        # No event loop exists yet
        asyncio.run(_run_all())

    elapsed = (time.time() - t_start) * 1000
    cached = len(runtime.speaker_global_cache)
    total = len(SPEAKER_MAP)
    logger.info(f"Speaker globals cached: {cached}/{total} speakers in {elapsed:.0f}ms")

async def generate_speech(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    output_sample_rate: str = "16khz",
    request_id: Optional[str] = None,
) -> Tuple[Optional[bytes], Dict[str, Any]]:
    """
    Generate speech audio (non-streaming).

    Args:
        text: Text to synthesize (already normalized)
        speaker: Internal speaker name (resolved)
        temperature: Sampling temperature
        top_k: Top-k sampling
        top_p: Nucleus sampling
        max_tokens: Maximum tokens to generate
        repetition_penalty: Repetition penalty
        seed: Random seed for reproducibility
        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)

    Returns:
        Tuple of (audio_bytes, metrics_dict)
        audio_bytes: WAV audio bytes (16kHz or 48kHz, 16-bit PCM) or None if failed
        metrics_dict: Dictionary with timing metrics

    Raises:
        RuntimeError: If runtime not initialized
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()
    metrics = {
        "request_id": request_id,
        "ttfb_ms": 0,
        "generation_ms": 0,
        "tokens_generated": 0,
        "audio_duration_seconds": 0.0,
        "sr_applied": False,
        "output_sample_rate": 16000,
    }

    if request_id is None:
        import uuid
        request_id = f"runtime-ns-{uuid.uuid4().hex[:10]}"
        metrics["request_id"] = request_id

    active_requests = _with_inflight_delta(1)
    start_time = time.time()

    pipeline_timings = {}

    try:
        logger.info(
            "tts_runtime_nonstream_start",
            extra={
                "speaker": speaker,
                "request_id": request_id,
                "text_length": len(text),
                "request_inflight": active_requests,
                "temperature": temperature,
                "top_k": top_k,
                "top_p": top_p,
                "max_tokens": max_tokens,
            },
        )

        gen_kwargs = dict(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
        )

        if _timing_enabled():
            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
                **gen_kwargs,
                return_timing=True,
                request_id=request_id,
            )
        else:
            audio_bytes = await runtime.pipeline.generate_speech_indic(
                **gen_kwargs,
                request_id=request_id,
            )

        generation_time = time.time() - start_time
        metrics["generation_ms"] = int(generation_time * 1000)
        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time

        if pipeline_timings:
            metrics.update({
                "llm_token_total": pipeline_timings.get("token_total", 0),
                "llm_batch_count": pipeline_timings.get("batch_count", 0),
                "llm_batch_gpu_ms_total": pipeline_timings.get("llm_gpu_batch_ms_total", 0),
                "llm_batch_gpu_ms_min": pipeline_timings.get("llm_gpu_batch_ms_min", 0),
                "llm_batch_gpu_ms_max": pipeline_timings.get("llm_gpu_batch_ms_max", 0),
                "llm_batch_gpu_ms_p50": pipeline_timings.get("llm_gpu_batch_ms_p50", 0),
                "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
                "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
                "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
                "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
                "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
                "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
            })

        if audio_bytes:
            # Apply super-resolution if requested and available
            sample_rate = 16000
            logger.info(f"SR check: output_sample_rate={output_sample_rate!r}, "
                       f"sr_service={runtime.sr_service is not None}, "
                       f"is_loaded={runtime.sr_service.is_loaded if runtime.sr_service else False}")

            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
                try:
                    sr_start = time.time()
                    logger.info(f"Applying super-resolution to {len(audio_bytes)} bytes...")
                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
                    sr_time = (time.time() - sr_start) * 1000
                    metrics["sr_ms"] = int(sr_time)
                    metrics["sr_applied"] = True
                    sample_rate = 48000
                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
                except Exception as e:
                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
                    import traceback
                    traceback.print_exc()

            metrics["output_sample_rate"] = sample_rate

            # Calculate audio duration
                audio_duration = (len(audio_bytes) - 44) / (sample_rate * 2)  # -44 for WAV header
                metrics["audio_duration_seconds"] = max(0.0, audio_duration)

        if _timing_enabled():
            logger.info("tts_runtime_nonstream_complete", extra=metrics)

        return audio_bytes, metrics

    except Exception as e:
        logger.error(f"Speech generation failed: {e}")
        raise
        finally:
            active_requests = _with_inflight_delta(-1)
            if _timing_enabled():
                logger.info(
                    "tts_runtime_nonstream_inflight",
                    extra={"active_requests": active_requests},
                )

def _apply_super_resolution(audio_bytes: bytes, sr_service) -> bytes:
    """
    Apply super-resolution to audio bytes.

    Args:
        audio_bytes: WAV audio at 16kHz
        sr_service: SuperResolutionService instance

    Returns:
        WAV audio at 48kHz
    """
    import numpy as np
    import struct
    import torch

    # Parse WAV header and extract PCM data
    if len(audio_bytes) < 44:
        raise ValueError("Invalid WAV data")

    # Extract PCM data (skip 44-byte WAV header)
    pcm_data = np.frombuffer(audio_bytes[44:], dtype=np.int16)

    # Convert to float32 for SR model
    audio_float = pcm_data.astype(np.float32) / 32768.0

    # Convert numpy to torch tensor for process_chunk
    audio_tensor = torch.from_numpy(audio_float)

    # Apply super-resolution using process_chunk (16kHz -> 48kHz)
    # process_chunk expects [batch, samples] or [samples] and returns [batch, samples]
    upsampled_tensor = sr_service.process_chunk(audio_tensor)

    # Convert back to numpy, squeeze batch dim if present
    upsampled = upsampled_tensor.squeeze().cpu().numpy()


Output:
Chunk ID: 9e9695
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2916
Output:
"""
Spark TTS Generation Pipeline

End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
"""

import asyncio
import logging
import os
import re
from statistics import mean, median
from typing import Optional, List, Dict, Any, Tuple

from vllm import SamplingParams
import torch

logger = logging.getLogger(__name__)

from veena3modal.core.constants import (
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_SEED,
    AUDIO_SAMPLE_RATE,
)

def _detailed_timing_enabled() -> bool:
    """
    Return True when benchmark logs are enabled.

    Controlled by environment variable:
      VEENA3_TIMING_LOGS=true|1
    """
    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}

class SparkTTSPipeline:
    """
    End-to-end TTS pipeline for Spark TTS with BiCodec.

    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        bicodec_decoder,
    ):
        """
        Initialize pipeline.

        Args:
            model: SparkTTS Model instance
            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
            bicodec_decoder: BiCodecDecoder instance
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.bicodec_decoder = bicodec_decoder

        print(f"🚀 SparkTTSPipeline initialized")

    async def generate_speech(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
        """
        Generate speech audio (non-streaming) using Spark TTS.

        NOTE: This method signature changed from description-based to speaker-based
        to align with Spark TTS architecture.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize (with optional [emotion] tags)
            temperature: Sampling temperature (default: 0.8 for Spark TTS)
            top_k: Top-k sampling (default: 50)
            top_p: Nucleus sampling (default: 1.0)
            max_tokens: Max BiCodec tokens to generate (default: 2048)
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Build prompt using Spark TTS format
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling for Spark TTS
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
        )

        # Use vLLM engine to generate
        if request_id is None:
            import uuid
            request_id = f"req-{uuid.uuid4().hex[:12]}"

        emit_timing = return_timing or _detailed_timing_enabled()
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Collect all outputs and track generation timing
        final_output = None
        token_batches: List[int] = []
        batch_wall_ms: List[float] = []
        batch_gpu_ms: List[float] = []
        total_token_ids = 0
        first_token_ms: Optional[float] = None
        t_start = asyncio.get_event_loop().time()

        iterator = results_generator.__aiter__()
        while True:
            wall_start = asyncio.get_event_loop().time()
            start_event = None
            end_event = None
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                start_event = torch.cuda.Event(enable_timing=True)
                end_event = torch.cuda.Event(enable_timing=True)
                start_event.record()

            try:
                request_output = await iterator.__anext__()
            except StopAsyncIteration:
                break

            wall_end = asyncio.get_event_loop().time()
            batch_ms = (wall_end - wall_start) * 1000
            batch_wall_ms.append(batch_ms)

            batch_gpu_ms_value: Optional[float] = None
            if start_event is not None and end_event is not None:
                end_event.record()
                torch.cuda.synchronize()
                batch_gpu_ms_value = start_event.elapsed_time(end_event)
            if batch_gpu_ms_value is not None:
                batch_gpu_ms.append(batch_gpu_ms_value)

            total_generated = len(request_output.outputs[0].token_ids)
            new_tokens = max(0, total_generated - total_token_ids)
            total_token_ids = total_generated
            token_batches.append(new_tokens)

            if first_token_ms is None:
                first_token_ms = (wall_end - t_start) * 1000

            if emit_timing:
                logger.info(
                    "tts_llm_batch",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "batch_index": len(token_batches),
                        "batch_tokens": new_tokens,
                        "llm_batch_wall_ms": round(batch_ms, 3),
                        "llm_batch_gpu_ms": round(batch_gpu_ms_value, 3)
                        if batch_gpu_ms_value is not None
                        else None,
                    },
                )
            final_output = request_output

        if final_output is None:
            if emit_timing:
                logger.warning(
                    "tts_llm_no_output",
                    extra={"speaker": speaker, "request_id": request_id},
                )
            print(f"❌ Generation failed: no output")
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": 0,
                    "token_total": 0,
                    "batch_count": 0,
                }
            return None

        # Extract generated text (contains BiCodec tokens)
        parse_start = asyncio.get_event_loop().time()
        generated_text = final_output.outputs[0].text
        generated_ids = final_output.outputs[0].token_ids

        # Extract BiCodec tokens using regex
        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)

        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000

        if not semantic_ids or not global_ids:
            if emit_timing:
                logger.error(
                    "tts_bicodec_parse_failed",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "semantic_count": len(semantic_ids),
                        "global_count": len(global_ids),
                        "token_total": len(generated_ids),
                    },
                )
            logger.error(
                "❌ No BiCodec tokens found in output",
                extra={
                    "speaker": speaker,
                    "text_length": len(text),
                    "text_preview": text[:100],
                    "generated_tokens": len(generated_ids),
                    "semantic_tokens": len(semantic_ids),
                    "global_tokens": len(global_ids),
                    "generated_text_preview": generated_text[:500],
                }
            )
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Validate tokens
        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
            logger.error("❌ BiCodec token validation failed", extra={
                "speaker": speaker,
                "text_length": len(text),
                "semantic_tokens": len(semantic_ids),
                "global_tokens": len(global_ids),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Decode to audio
        decode_start = asyncio.get_event_loop().time()
        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000

        if audio_bytes is None:
            if emit_timing:
                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
            logger.error("❌ BiCodec decode failed", extra={
                "speaker": speaker,
                "text_length": len(text),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        total_ms = (asyncio.get_event_loop().time() - t_start) * 1000

        timing_metrics: Dict[str, Any] = {
            "request_id": request_id,
            "speaker": speaker,
            "text_length": len(text),
            "batch_count": len(token_batches),
            "token_total": len(generated_ids),
            "token_batches": token_batches,
            "ttfb_ms": int(first_token_ms or 0),
            "llm_wall_ms": int(sum(batch_wall_ms)),
            "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
            "llm_gpu_batch_ms_total": int(sum(batch_gpu_ms)),
            "llm_gpu_batch_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_p50": int(median(batch_gpu_ms)) if batch_gpu_ms else 0,
            "decode_cpu_ms": int(decode_ms),
            "parse_ms": int(parse_ms),
            "total_ms": int(total_ms),
        }
        if token_batches:
            timing_metrics["tokens_per_batch_min"] = min(token_batches)
            timing_metrics["tokens_per_batch_max"] = max(token_batches)
            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))

        if emit_timing:
            logger.info("tts_llm_complete", extra=timing_metrics)

        from veena3modal.audio.utils import add_wav_header
        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)

        if return_timing:
            return wav_bytes, timing_metrics
        return wav_bytes


Output:
Chunk ID: abd468
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2715
Output:
"""
Veena3 Streaming Pipeline - Sliding Window Approach

Implements streaming for both SNAC and BiCodec tokens.

For BiCodec (Spark TTS):
1. Generate tokens from vLLM
2. Parse NEW token IDs incrementally (O(1) per token via cache)
3. Buffer semantic + global tokens
4. Apply sliding window (every N token pairs)
5. Decode and stream audio chunks

OPTIMIZATION (Dec 2025):
- Replaced O(n²) pattern (decode-all + regex-all per iteration)
- Now uses incremental token parsing with BiCodecTokenParser
- ~10x CPU reduction in streaming hot loop

For SNAC (legacy):
1. Filter SNAC token IDs directly
2. Apply sliding window
3. Decode and stream
"""

import asyncio
import os
import re
from statistics import median, mean
import torch
from typing import AsyncGenerator, Optional, List
from vllm import SamplingParams
from veena3modal.audio.crossfade import crossfade_bytes_int16
from veena3modal.core.token_utils import BiCodecTokenParser

from veena3modal.core.constants import (
    CODE_END_TOKEN_ID,
    CODE_START_TOKEN_ID,
    SNAC_MIN_ID,
    SNAC_MAX_ID,
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_MIN_TOKENS,
    DEFAULT_REPETITION_PENALTY,
    DEFAULT_SEED,
)

def _detailed_timing_enabled() -> bool:
    """Enable per-batch token/decode timing logs."""
    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}

class Veena3SlidingWindowPipeline:
    """
    Streaming TTS pipeline using sliding window approach.

    This eliminates choppy audio and popping artifacts by:
    - Decoding overlapping 28-token windows (4 frames)
    - Keeping only the middle 2048 samples from each decode
    - Creating natural continuity between chunks

    Based on the official Canopy Labs implementation.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        snac_decoder,
    ):
        """
        Initialize sliding window streaming pipeline.

        Args:
            model: Veena3Model instance
            prompt_builder: Veena3PromptBuilder instance
            snac_decoder: SNACDecoder instance (with batching enabled)
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.snac_decoder = snac_decoder

        # OPTIMIZATION: Pre-warm BiCodecTokenParser once at init, not per-request
        # Saves ~123ms per streaming TTFB by avoiding repeated 166K-entry vocab iteration
        tokenizer = getattr(model, "tokenizer", None)
        if tokenizer is None:
            tokenizer = getattr(model.engine, "tokenizer", None)
        self.token_parser = BiCodecTokenParser(tokenizer) if tokenizer else None

        print(f"🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)")

    async def generate_speech_stream(
        self,
        description: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech audio with sliding window streaming.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            description: Character/voice description
            text: Text to synthesize (with optional <emotion> tags)
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops

        Yields:
            Audio bytes (int16 PCM, 24kHz mono)
        """
        print(f"\n🌊 Sliding window streaming generation")
        print(f"📝 Description: {description[:80]}...")
        print(f"💬 Text: {text}")

        # Build prompt
        prompt = self.prompt_builder.build_prefix(description, text)
        print(f"✅ Prompt built ({len(prompt)} chars)")

        # Configure sampling
        sampling_params = SamplingParams(
            temperature=temperature,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop_token_ids=[CODE_END_TOKEN_ID],  # Only audio EOS
            seed=seed,  # None = random, int = reproducible
        )

        print(f"🎲 Sampling: temp={temperature}, top_p={top_p}, sliding_window=28 tokens")

        # Token buffer - keeps ALL tokens (not chunked)
        token_buffer = []
        total_tokens_generated = 0
        total_audio_chunks = 0

        # Generate tokens with vLLM (streaming)
        print(f"🔮 Starting token generation...")

        # Generate unique request ID for concurrent streaming support
        import uuid
        import time
        request_id = f"slide-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Stream tokens with sliding window
        async for request_output in results_generator:
            # Extract generated token IDs
            generated_ids = request_output.outputs[0].token_ids

            # Process only new tokens
            new_tokens = generated_ids[total_tokens_generated:]
            total_tokens_generated = len(generated_ids)

            # Filter and buffer SNAC tokens
            for token_id in new_tokens:
                if SNAC_MIN_ID <= token_id <= SNAC_MAX_ID:
                    token_buffer.append(token_id)

                    # Process every 7 tokens (1 frame) when we have enough for sliding window
                    # Official approach: once count > 27, take last 28 tokens
                    if len(token_buffer) % 7 == 0 and len(token_buffer) > 27:
                        # Sliding window: take last 28 tokens (4 frames)
                        window_tokens = token_buffer[-28:]

                        # Decode with sliding window mode (returns middle 2048 samples only)
                        if self.snac_decoder.enable_batching:
                            audio_bytes = await self.snac_decoder.decode_single_async(
                                window_tokens,
                                trim_warmup=False,  # Sliding window handles trimming
                                use_sliding_window=True  # CRITICAL: Use sliding window mode
                            )
                        else:
                            audio_bytes = self.snac_decoder.decode_to_bytes(
                                window_tokens,
                                trim_warmup=False,
                                use_sliding_window=True
                            )

                        if audio_bytes:
                            total_audio_chunks += 1
                            if total_audio_chunks == 1:
                                print(f"🎵 First chunk decoded ({len(audio_bytes)} bytes, sliding window)")
                            yield audio_bytes

        # Note: No final chunk processing needed - sliding window handles all tokens
        # as they come in (every 7 tokens after the first 28)

        print(f"✅ Sliding window streaming complete: {total_tokens_generated} tokens → {total_audio_chunks} audio chunks")

    def _extract_bicodec_tokens_from_text(self, text: str) -> tuple[List[int], List[int]]:
        """
        Extract BiCodec semantic and global tokens from generated text.

        Args:
            text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
        """
        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", text)
        semantic_ids = [int(t) for t in semantic_matches]

        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", text)
        global_ids = [int(t) for t in global_matches]

        return semantic_ids, global_ids

    async def generate_speech_stream_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,  # Added for Spark TTS parity with non-streaming
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        request_id: Optional[str] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech audio with sliding window streaming for Indic model.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "<laugh> Hello there!"
                - "नमस्ते! <excited> आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops
            seed: Random seed for reproducibility

        Yields:
            Audio bytes (int16 PCM, 16kHz mono - BiCodec)
        """
        import time
        t_start = time.time()
        emit_timing = _detailed_timing_enabled()

        # Build prompt using Indic prompt builder
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling (matching non-streaming pipeline parameters)
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID (128258)
        # which doesn't exist in the Spark vocab and would cause generation to run to max_tokens
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,  # Keep BiCodec tokens in output
            seed=seed,
        )

        # BiCodec token buffers (separate for semantic and global)
        semantic_buffer = []
        global_buffer = []
        processed_token_count = 0  # Track how many vLLM tokens we've parsed


cd /home/ubuntu/veenaModal && sed -n '280,620p' veena3modal/core/streaming_pipeline.py