Output:
Chunk ID: 3d97b3
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 2837
Output:
"""
Modal entrypoint for Veena3 TTS service.

Deploys a GPU-backed ASGI app with:
- True streaming TTS
- Autoscaling (0 to N containers)
- Per-container concurrency (vLLM batching)
- Memory snapshots for fast cold starts

Usage:
    # Deploy to Modal
    modal deploy veena3modal/app.py

    # Serve locally (for testing)
    modal serve veena3modal/app.py

    # Run a single function
    modal run veena3modal/app.py::tts_api
"""

from __future__ import annotations

import os
import modal

# Get the repo root for adding local sources
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# === Image Definition ===
# Full dependencies for TTS inference

image = (
    modal.Image.debian_slim(python_version="3.10")
    .apt_install(
        "ffmpeg",  # Audio encoding (opus, mp3, flac, mulaw)
        "git",     # For pip git installs
    )
    .pip_install(
        # Core ML/AI
        "torch>=2.0.0",
        "torchaudio>=2.0.0",
        "transformers>=4.35.0",
        "vllm>=0.4.0",

        # FastAPI
        "fastapi[standard]>=0.100.0",
        "uvicorn[standard]>=0.23.0",

        # Audio processing
        "numpy>=1.24.0",
        "scipy>=1.10.0",
        "librosa>=0.10.0",
        "soundfile>=0.12.0",

        # Utils
        "pydantic>=2.0.0",
        "prometheus_client>=0.17.0",
        "supabase>=2.0.0",

        # vLLM dependencies
        "einops>=0.6.0",
        "einx>=0.2.0",

        # SparkTTS dependencies (from external/sparktts/requirements.txt)
        "omegaconf>=2.3.0",
        "safetensors>=0.5.0",
        "soxr>=0.5.0",

        # AP-BWE (super-resolution) dependencies
        "matplotlib>=3.8.0",
        "natsort>=8.0.0",
        "joblib>=1.0.0",
    )
    .run_commands(
        # Ensure CUDA is properly configured
        "pip install --no-cache-dir nvidia-ml-py3 || true"
    )
    # Environment settings
    .env({
        "AUTH_BYPASS_MODE": "true",  # Disable auth for testing
        # sparktts is at /root/external/sparktts, AP-BWE at /root/external/AP-BWE
        "PYTHONPATH": "/root/external/sparktts:/root/external/AP-BWE:/root",
        "_IMAGE_BUILD_VERSION": "2025-12-25-sr-chunking-support",  # Add SR support to generate_speech_chunked
    })
    # Add sparktts package - create proper structure by copying files
    # The original has symlinks that don't work with Modal mounts
    .run_commands(
        # Create sparktts package directory
        "mkdir -p /root/external/sparktts/sparktts/models",
        "mkdir -p /root/external/sparktts/sparktts/utils",
    )
    .add_local_dir(
        local_path=os.path.join(REPO_ROOT, "external", "sparktts", "sparktts"),
        remote_path="/root/external/sparktts/sparktts",
        copy=True,
    )
    .add_local_dir(
        local_path=os.path.join(REPO_ROOT, "external", "sparktts", "models"),
        remote_path="/root/external/sparktts/sparktts/models",  # Put models inside sparktts package
        copy=True,
    )
    .add_local_dir(
        local_path=os.path.join(REPO_ROOT, "external", "sparktts", "utils"),
        remote_path="/root/external/sparktts/sparktts/utils",  # Put utils inside sparktts package
        copy=True,
    )
    # Add AP-BWE for super-resolution (16kHz -> 48kHz)
    .add_local_dir(
        local_path=os.path.join(REPO_ROOT, "external", "AP-BWE"),
        remote_path="/root/external/AP-BWE",
        copy=True,
    )
    # Non-copy local mounts (MUST BE LAST - no build steps after this!)
    .add_local_python_source("veena3modal")
)

# === Volume for Model Weights ===
# Models should be uploaded to this volume before deployment

try:
    model_volume = modal.Volume.from_name("veena3-models", create_if_missing=True)
except Exception:
    model_volume = None  # Will fail at runtime if volume not available

# === Secrets (Optional) ===
# Secrets are optional - the app will run in bypass mode without them
# Create secrets in Modal dashboard: https://modal.com/secrets
# Or use modal.Secret.from_dotenv() for local testing

# === App Definition ===

app = modal.App(
    name="veena3-tts",
    image=image,
)

@app.cls(
    # === GPU Selection ===
    # L40S: Best cost/performance ratio for TTS inference
    # - 48GB VRAM, $1.50/hr (vs A100-80GB at $4.50/hr)
    # - Handles ~12 req/s with <2s p95 latency per container
    # - Memory-bound workload, so faster GPUs have diminishing returns
    gpu="L40S",

    # === CPU Allocation (OPTIMIZATION Dec 2025) ===
    # Explicitly reserve 4 vCPUs for the Python driver process
    # Prevents CPU throttling during vLLM scheduler + streaming loop
    # Modal default is only 0.125 cores which can bottleneck streaming
    cpu=4.0,

    volumes={"/models": model_volume} if model_volume else {},
    secrets=[modal.Secret.from_name("veena3-secrets")],  # Supabase credentials

    # === Autoscaling Configuration ===
    # See .cursor/scaling.md for detailed analysis
    #
    # LOAD TEST RESULTS (Dec 2025):
    # - Cold boot: ~49s (memory snapshot captures imports, not GPU model)
    # - MAX SUSTAINABLE: 280 req/s @ 250 concurrent users (P95=1417ms, 100% success)
    # - Breaking point: 300+ users (429 errors start)
    # - Per container: ~14 req/s (280/20 containers)
    #
    min_containers=1,        # Keep 1 warm 24/7 for instant response
    max_containers=50,       # ~280 req/s capacity at P95 < 3s
    buffer_containers=1,     # REQUIRED: 49s cold boot too slow for burst
    scaledown_window=300,    # 5 min idle before scaledown (user changed from 120)

    # === Timeouts ===
    timeout=120,             # 2 min max per request (reduced from 10 min)
    startup_timeout=600,     # 10 min for model loading (reduced from 20 min)

    # === Memory Optimization ===
    # Memory snapshot for faster cold boots
    # Note: GPU snapshot is experimental (alpha) - disabled for stability
    enable_memory_snapshot=True,
)
# === Per-Container Concurrency ===
# vLLM continuous batching handles concurrent requests efficiently
# Increased from 12 to 20 to maximize GPU utilization while keeping P95 < 3s
@modal.concurrent(max_inputs=20, target_inputs=15)
class TTSService:
    """
    TTS service class with model lifecycle management.

    Uses @modal.enter for model loading (once per container).
    Memory snapshot captures imports and non-GPU state for faster restarts.
    """

    @modal.enter()
    def load_model(self):
        """
        Load TTS model when container starts.

        Called once per container lifecycle.
        Memory snapshot captures import state but not GPU memory.
        """
        import os
        import logging

        logging.basicConfig(level=logging.INFO)
        logger = logging.getLogger(__name__)

        logger.info("🚀 Starting TTS model loading...")

        # Set model paths from Volume mount or env vars
        model_path = os.environ.get(
            "MODEL_PATH",
            "/models/spark_tts_4speaker"
        )
        sr_path = os.environ.get(
            "AP_BWE_CHECKPOINT_DIR",
            "/models/ap_bwe/16kto48k"
        )

        # Check if model exists
        if not os.path.exists(model_path):
            logger.warning(f"Model path not found: {model_path}")
            logger.warning("TTS will return 503 until model is available")
            return

        try:
            from veena3modal.services.tts_runtime import initialize_runtime

            # Initialize with GPU memory optimization
            self.runtime = initialize_runtime(
                model_path=model_path,
                sr_checkpoint_dir=sr_path if os.path.exists(sr_path) else None,
                device="cuda",
                gpu_memory_utilization=0.85,
                enable_sr=os.path.exists(sr_path),
            )

            logger.info(f"✅ TTS model loaded: {self.runtime.model_version}")

        except Exception as e:
            logger.error(f"❌ Model loading failed: {e}")
            import traceback
            traceback.print_exc()

    @modal.asgi_app()
    def serve(self):
        """
        Return FastAPI ASGI app for serving requests.
        """
        from veena3modal.api.fastapi_app import create_app
        return create_app()

# === Standalone Function (Alternative Deployment) ===
# Use this if you don't need @modal.enter lifecycle
# Note: Prefer TTSService class for production (better model lifecycle management)

@app.function(
    gpu="L40S",  # Best cost/performance for TTS
    cpu=4.0,     # Explicit CPU allocation (OPTIMIZATION Dec 2025)
    volumes={"/models": model_volume} if model_volume else {},
    secrets=[modal.Secret.from_name("veena3-secrets")],
    min_containers=0,        # Scale to zero (no min for standalone)
    max_containers=5,        # Lower cap for standalone
    buffer_containers=1,
    scaledown_window=180,    # 3 min
    timeout=120,             # 2 min max
    startup_timeout=600,     # 10 min for model load
)
@modal.asgi_app()
def tts_api():
    """
    Standalone ASGI app function.

    Note: Model will be loaded on first request (higher TTFB).
    For production, use TTSService class with @modal.enter.
    """
    from veena3modal.api.fastapi_app import create_app
    return create_app()

# === Health Check Function ===

@app.function(
    # No GPU needed for health check
)
def health_check():
    """
    Simple health check that can be called from Modal dashboard.
    """
    return {
        "status": "ok",
        "service": "veena3-tts",
        "version": "0.1.0",
    }

# === Debug Function ===

@app.function(
    gpu="L40S",
    volumes={"/models": model_volume} if model_volume else {},
    timeout=300,
)
def debug_imports():
    """
    Debug function to test import paths and file structure.
    Run with: modal run veena3modal/app.py::debug_imports
    """
    import os
    import sys

    results = {
        "pythonpath": os.environ.get("PYTHONPATH", "NOT_SET"),
        "sys_path": sys.path[:10],  # First 10 entries
        "files": {},
        "import_tests": {},
    }

    # Check key directories
    dirs_to_check = [
        "/root",
        "/root/external",
        "/root/external/sparktts",
        "/root/external/sparktts/sparktts",
        "/root/external/sparktts/sparktts/models",
        "/root/veena3modal",
        "/models",
    ]

    for d in dirs_to_check:
        if os.path.exists(d):
            try:
                files = os.listdir(d)
                results["files"][d] = files[:20]  # First 20 files
            except Exception as e:
                results["files"][d] = f"ERROR: {e}"
        else:
            results["files"][d] = "DOES_NOT_EXIST"

    # Test imports
    import_tests = [
        "sparktts",
        "sparktts.models",
        "sparktts.models.audio_tokenizer",
        "veena3modal",
        "veena3modal.core",
        "veena3modal.processing",
        "veena3modal.audio",
    ]

    for module in import_tests:
        try:
            __import__(module)
            results["import_tests"][module] = "OK"
        except Exception as e:
            results["import_tests"][module] = f"FAILED: {e}"

    return results

# === Local Testing Entry Point ===

@app.local_entrypoint()


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


cd /home/ubuntu/veena3Modal && sed -n '360,520p' veena3modal/app.py


Output:
Chunk ID: 313e2d
Wall time: 0.0517 seconds
Process exited with code 1
Original token count: 19
Output:
/bin/bash: line 1: cd: /home/ubuntu/veena3Modal: No such file or directory


Summary: auto


cd /home/ubuntu/veenaModal && nl -ba veena3modal/app.py | sed -n '360,520p'


Output:
Chunk ID: 0a6bdc
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 124
Output:
   360	@app.local_entrypoint()
   361	def main():
   362	    """
   363	    Local entrypoint for testing.
   364
   365	    Usage: modal run veena3modal/app.py
   366	    """
   367	    import json
   368
   369	    print("Veena3 TTS Modal App")
   370	    print("=" * 40)
   371	    print("\n🔍 Running debug_imports on GPU container...")
   372
   373	    result = debug_imports.remote()
   374	    print("\n📊 Debug Results:")
   375	    print(json.dumps(result, indent=2))


cd /home/ubuntu/veenaModal && [ -f AGENTS.md ] && sed -n '1,260p' AGENTS.md


Output:
Chunk ID: 8528ef
Wall time: 0.0513 seconds
Process exited with code 1
Original token count: 0
Output:


cd /home/ubuntu/veenaModal && head -n 80 veena3modal/core/pipeline.py


Output:
Chunk ID: bc34d0
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 510
Output:
"""
Spark TTS Generation Pipeline

End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
"""

import asyncio
import logging
import os
import re
from statistics import mean, median
from typing import Optional, List, Dict, Any, Tuple

from vllm import SamplingParams
import torch

logger = logging.getLogger(__name__)

from veena3modal.core.constants import (
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_SEED,
    AUDIO_SAMPLE_RATE,
)

def _detailed_timing_enabled() -> bool:
    """
    Return True when benchmark logs are enabled.

    Controlled by environment variable:
      VEENA3_TIMING_LOGS=true|1
    """
    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}

class SparkTTSPipeline:
    """
    End-to-end TTS pipeline for Spark TTS with BiCodec.

    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        bicodec_decoder,
    ):
        """
        Initialize pipeline.

        Args:
            model: SparkTTS Model instance
            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
            bicodec_decoder: BiCodecDecoder instance
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.bicodec_decoder = bicodec_decoder

        print(f"🚀 SparkTTSPipeline initialized")

    async def generate_speech(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:


cd /home/ubuntu/veenaModal && nl -ba veena3modal/services/tts_runtime.py | sed -n '420,980p'


Output:
Chunk ID: ad4273
Wall time: 0.0532 seconds
Process exited with code 0
Original token count: 6277
Output:
   420
   421	    pipeline_timings = {}
   422
   423	    try:
   424	        logger.info(
   425	            "tts_runtime_nonstream_start",
   426	            extra={
   427	                "speaker": speaker,
   428	                "request_id": request_id,
   429	                "text_length": len(text),
   430	                "request_inflight": active_requests,
   431	                "temperature": temperature,
   432	                "top_k": top_k,
   433	                "top_p": top_p,
   434	                "max_tokens": max_tokens,
   435	            },
   436	        )
   437
   438	        gen_kwargs = dict(
   439	            speaker=speaker,
   440	            text=text,
   441	            temperature=temperature,
   442	            top_k=top_k,
   443	            top_p=top_p,
   444	            max_tokens=max_tokens,
   445	            repetition_penalty=repetition_penalty,
   446	            seed=seed,
   447	        )
   448
   449	        if _timing_enabled():
   450	            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
   451	                **gen_kwargs,
   452	                return_timing=True,
   453	                request_id=request_id,
   454	            )
   455	        else:
   456	            audio_bytes = await runtime.pipeline.generate_speech_indic(
   457	                **gen_kwargs,
   458	                request_id=request_id,
   459	            )
   460
   461	        generation_time = time.time() - start_time
   462	        metrics["generation_ms"] = int(generation_time * 1000)
   463	        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time
   464
   465	        if pipeline_timings:
   466	            metrics.update({
   467	                "llm_token_total": pipeline_timings.get("token_total", 0),
   468	                "llm_batch_count": pipeline_timings.get("batch_count", 0),
   469	                "llm_batch_gpu_ms_total": pipeline_timings.get("llm_gpu_batch_ms_total", 0),
   470	                "llm_batch_gpu_ms_min": pipeline_timings.get("llm_gpu_batch_ms_min", 0),
   471	                "llm_batch_gpu_ms_max": pipeline_timings.get("llm_gpu_batch_ms_max", 0),
   472	                "llm_batch_gpu_ms_p50": pipeline_timings.get("llm_gpu_batch_ms_p50", 0),
   473	                "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
   474	                "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
   475	                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
   476	                "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
   477	                "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
   478	                "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
   479	                "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
   480	            })
   481
   482	        if audio_bytes:
   483	            # Apply super-resolution if requested and available
   484	            sample_rate = 16000
   485	            logger.info(f"SR check: output_sample_rate={output_sample_rate!r}, "
   486	                       f"sr_service={runtime.sr_service is not None}, "
   487	                       f"is_loaded={runtime.sr_service.is_loaded if runtime.sr_service else False}")
   488
   489	            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
   490	                try:
   491	                    sr_start = time.time()
   492	                    logger.info(f"Applying super-resolution to {len(audio_bytes)} bytes...")
   493	                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
   494	                    sr_time = (time.time() - sr_start) * 1000
   495	                    metrics["sr_ms"] = int(sr_time)
   496	                    metrics["sr_applied"] = True
   497	                    sample_rate = 48000
   498	                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
   499	                except Exception as e:
   500	                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
   501	                    import traceback
   502	                    traceback.print_exc()
   503
   504	            metrics["output_sample_rate"] = sample_rate
   505
   506	            # Calculate audio duration
   507	            audio_duration = (len(audio_bytes) - 44) / (sample_rate * 2)  # -44 for WAV header
   508	            metrics["audio_duration_seconds"] = max(0.0, audio_duration)
   509
   510	        if _timing_enabled():
   511	            logger.info("tts_runtime_nonstream_complete", extra=metrics)
   512
   513	        return audio_bytes, metrics
   514
   515	    except Exception as e:
   516	        logger.error(f"Speech generation failed: {e}")
   517	        raise
   518	    finally:
   519	        active_requests = _with_inflight_delta(-1)
   520	        if _timing_enabled():
   521	            logger.info(
   522	                "tts_runtime_nonstream_inflight",
   523	                extra={
   524	                    "request_id": request_id,
   525	                    "active_requests": active_requests,
   526	                },
   527	            )
   528
   529
   530	def _apply_super_resolution(audio_bytes: bytes, sr_service) -> bytes:
   531	    """
   532	    Apply super-resolution to audio bytes.
   533
   534	    Args:
   535	        audio_bytes: WAV audio at 16kHz
   536	        sr_service: SuperResolutionService instance
   537
   538	    Returns:
   539	        WAV audio at 48kHz
   540	    """
   541	    import numpy as np
   542	    import struct
   543	    import torch
   544
   545	    # Parse WAV header and extract PCM data
   546	    if len(audio_bytes) < 44:
   547	        raise ValueError("Invalid WAV data")
   548
   549	    # Extract PCM data (skip 44-byte WAV header)
   550	    pcm_data = np.frombuffer(audio_bytes[44:], dtype=np.int16)
   551
   552	    # Convert to float32 for SR model
   553	    audio_float = pcm_data.astype(np.float32) / 32768.0
   554
   555	    # Convert numpy to torch tensor for process_chunk
   556	    audio_tensor = torch.from_numpy(audio_float)
   557
   558	    # Apply super-resolution using process_chunk (16kHz -> 48kHz)
   559	    # process_chunk expects [batch, samples] or [samples] and returns [batch, samples]
   560	    upsampled_tensor = sr_service.process_chunk(audio_tensor)
   561
   562	    # Convert back to numpy, squeeze batch dim if present
   563	    upsampled = upsampled_tensor.squeeze().cpu().numpy()
   564
   565	    # Convert back to int16
   566	    upsampled_int16 = np.clip(upsampled * 32768.0, -32768, 32767).astype(np.int16)
   567
   568	    # Create new WAV header for 48kHz
   569	    sample_rate = 48000
   570	    num_channels = 1
   571	    bits_per_sample = 16
   572	    byte_rate = sample_rate * num_channels * bits_per_sample // 8
   573	    block_align = num_channels * bits_per_sample // 8
   574	    data_size = len(upsampled_int16) * 2
   575
   576	    wav_header = struct.pack(
   577	        '<4sI4s4sIHHIIHH4sI',
   578	        b'RIFF',
   579	        36 + data_size,
   580	        b'WAVE',
   581	        b'fmt ',
   582	        16,  # fmt chunk size
   583	        1,   # PCM format
   584	        num_channels,
   585	        sample_rate,
   586	        byte_rate,
   587	        block_align,
   588	        bits_per_sample,
   589	        b'data',
   590	        data_size,
   591	    )
   592
   593	    return wav_header + upsampled_int16.tobytes()
   594
   595
   596	async def generate_speech_chunked(
   597	    text: str,
   598	    speaker: str,
   599	    temperature: float = 0.8,
   600	    top_k: int = 50,
   601	    top_p: float = 1.0,
   602	    max_tokens: int = 4096,
   603	    repetition_penalty: float = 1.05,
   604	    seed: Optional[int] = None,
   605	    sample_rate: int = 16000,
   606	    output_sample_rate: str = "16khz",
   607	    request_id: Optional[str] = None,
   608	) -> Tuple[Optional[bytes], Dict[str, Any]]:
   609	    """
   610	    Generate speech with automatic text chunking for long inputs.
   611
   612	    Uses LongTextProcessor to split text and stitch audio.
   613
   614	    Args:
   615	        text: Text to synthesize
   616	        speaker: Internal speaker name
   617	        ... (same as generate_speech)
   618	        sample_rate: Internal generation sample rate (always 16000)
   619	        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)
   620
   621	    Returns:
   622	        Tuple of (audio_bytes, metrics_dict)
   623	    """
   624	    if not is_initialized():
   625	        raise RuntimeError("TTS runtime not initialized")
   626
   627	    runtime = get_runtime()
   628	    metrics = {
   629	        "ttfb_ms": 0,
   630	        "generation_ms": 0,
   631	        "chunks_processed": 0,
   632	        "audio_duration_seconds": 0.0,
   633	        "text_chunked": False,
   634	        "sr_applied": False,
   635	        "output_sample_rate": 16000,
   636	    }
   637
   638	    active_requests = _with_inflight_delta(1)
   639	    start_time = time.time()
   640
   641	    try:
   642	        if _timing_enabled():
   643	            logger.info(
   644	                "tts_runtime_chunking_start",
   645	                extra={
   646	                    "speaker": speaker,
   647	                    "text_length": len(text),
   648	                    "request_inflight": active_requests,
   649	                    "chunking": True,
   650	                },
   651	            )
   652	        # Import long text processor
   653	        from veena3modal.processing.long_text_processor import LongTextProcessor
   654
   655	        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
   656
   657	        # Check if chunking is needed
   658	        if long_processor.should_chunk(text):
   659	            metrics["text_chunked"] = True
   660	            audio_bytes = await long_processor.generate_with_chunking(
   661	                text=text,
   662	                speaker=speaker,
   663	                temperature=temperature,
   664	                top_k=top_k,
   665	                top_p=top_p,
   666	                max_tokens=max_tokens,
   667	                repetition_penalty=repetition_penalty,
   668	                seed=seed,
   669	                sample_rate=sample_rate,
   670	            )
   671	        else:
   672	            # Short text: direct generation
   673	            audio_bytes = await runtime.pipeline.generate_speech_indic(
   674	                speaker=speaker,
   675	                text=text,
   676	                temperature=temperature,
   677	                top_k=top_k,
   678	                top_p=top_p,
   679	                max_tokens=max_tokens,
   680	                repetition_penalty=repetition_penalty,
   681	                seed=seed,
   682	            )
   683
   684	        generation_time = time.time() - start_time
   685	        metrics["generation_ms"] = int(generation_time * 1000)
   686	        metrics["ttfb_ms"] = metrics["generation_ms"]
   687
   688	        if audio_bytes:
   689	            # Apply super-resolution if requested and available
   690	            final_sample_rate = 16000
   691	            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
   692	                try:
   693	                    sr_start = time.time()
   694	                    logger.info(f"Applying super-resolution to chunked audio ({len(audio_bytes)} bytes)...")
   695	                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
   696	                    sr_time = (time.time() - sr_start) * 1000
   697	                    metrics["sr_ms"] = int(sr_time)
   698	                    metrics["sr_applied"] = True
   699	                    final_sample_rate = 48000
   700	                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
   701	                except Exception as e:
   702	                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
   703	                    import traceback
   704	                    traceback.print_exc()
   705
   706	            metrics["output_sample_rate"] = final_sample_rate
   707	            audio_duration = (len(audio_bytes) - 44) / (final_sample_rate * 2)
   708	            metrics["audio_duration_seconds"] = max(0.0, audio_duration)
   709
   710	        if _timing_enabled():
   711	            metrics["active_requests"] = active_requests - 1
   712	            logger.info("tts_runtime_chunking_complete", extra=metrics)
   713	        return audio_bytes, metrics
   714
   715	    except Exception as e:
   716	        logger.error(f"Chunked speech generation failed: {e}")
   717	        raise
   718	    finally:
   719	        _with_inflight_delta(-1)
   720
   721
   722	from typing import AsyncGenerator
   723
   724
   725	async def generate_speech_streaming(
   726	    text: str,
   727	    speaker: str,
   728	    temperature: float = 0.8,
   729	    top_k: int = 50,
   730	    top_p: float = 1.0,
   731	    max_tokens: int = 4096,
   732	    repetition_penalty: float = 1.05,
   733	    seed: Optional[int] = None,
   734	    enable_chunking: bool = True,
   735	    request_id: Optional[str] = None,
   736	) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
   737	    """
   738	    Generate speech audio with true streaming (yields chunks as they're generated).
   739
   740	    This is the core streaming implementation for M4.
   741	    First yield includes WAV header + first PCM chunk.
   742	    Subsequent yields are raw PCM chunks.
   743
   744	    Args:
   745	        text: Text to synthesize (already normalized)
   746	        speaker: Internal speaker name (resolved)
   747	        temperature: Sampling temperature
   748	        top_k: Top-k sampling
   749	        top_p: Nucleus sampling
   750	        max_tokens: Maximum tokens to generate
   751	        repetition_penalty: Repetition penalty
   752	        seed: Random seed for reproducibility
   753	        enable_chunking: Enable text chunking for long inputs with voice consistency
   754
   755	    Yields:
   756	        Tuple of (audio_bytes, metrics_dict)
   757	        - First yield: WAV header (44 bytes) prepended to first PCM chunk
   758	        - Subsequent yields: Raw PCM chunks (int16, 16kHz)
   759	        - Final yield: metrics_dict has final timing info
   760
   761	    Raises:
   762	        RuntimeError: If runtime not initialized or streaming pipeline unavailable
   763	    """
   764	    if not is_initialized():
   765	        raise RuntimeError("TTS runtime not initialized")
   766
   767	    runtime = get_runtime()
   768
   769	    if runtime.streaming_pipeline is None:
   770	        raise RuntimeError("Streaming pipeline not available")
   771
   772	    # Import audio utils for WAV header
   773	    from veena3modal.audio.utils import create_wav_header
   774
   775	    # Metrics tracking
   776	    metrics = {
   777	        "ttfb_ms": 0,
   778	        "chunks_sent": 0,
   779	        "total_bytes": 0,
   780	        "audio_duration_seconds": 0.0,
   781	        "text_chunked": False,
   782	    }
   783
   784	    start_time = time.time()
   785	    first_chunk_time = None
   786	    total_pcm_bytes = 0
   787	    sample_rate = 16000  # BiCodec sample rate
   788	    active_requests = _with_inflight_delta(1)
   789	    if _timing_enabled():
   790	        logger.info(
   791	            "tts_runtime_stream_start",
   792	            extra={
   793	                "speaker": speaker,
   794	                "text_length": len(text),
   795	                "request_inflight": active_requests,
   796	                "chunking": enable_chunking,
   797	                "request_id": request_id,
   798	            },
   799	        )
   800
   801	    try:
   802	        # Check if we need text chunking
   803	        from veena3modal.processing.long_text_processor import LongTextProcessor
   804	        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
   805	        needs_chunking = enable_chunking and long_processor.should_chunk(text)
   806
   807	        if needs_chunking:
   808	            # Chunked streaming with voice consistency (global token caching)
   809	            metrics["text_chunked"] = True
   810	            async for audio_chunk, chunk_metrics in _stream_chunked_text(
   811	                runtime,
   812	                long_processor,
   813	                text,
   814	                speaker,
   815	                temperature,
   816	                top_k,
   817	                top_p,
   818	                max_tokens,
   819	                repetition_penalty,
   820	                seed,
   821	                sample_rate,
   822	                request_id,
   823	            ):
   824	                if first_chunk_time is None:
   825	                    first_chunk_time = time.time()
   826	                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
   827	                    # Create streaming WAV header (size=0 for unknown length)
   828	                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
   829	                    audio_chunk = wav_header + audio_chunk
   830
   831	                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
   832	                metrics["chunks_sent"] += 1
   833	                metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
   834	                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
   835	                if chunk_metrics:
   836	                    metrics.update(chunk_metrics)
   837
   838	                yield audio_chunk, metrics
   839	        else:
   840	            # Simple streaming (no text chunking)
   841	            # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
   842	            # use continuation mode to skip the ~110ms global token pre-roll.
   843	            # The model jumps straight to semantic token generation.
   844	            cached_globals = runtime.speaker_global_cache.get(speaker)
   845
   846	            if cached_globals:
   847	                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
   848	                    speaker=speaker,
   849	                    text=text,
   850	                    global_ids=cached_globals,
   851	                    temperature=temperature,
   852	                    top_k=top_k,
   853	                    top_p=top_p,
   854	                    max_tokens=max_tokens,
   855	                    repetition_penalty=repetition_penalty,
   856	                    seed=seed,
   857	                    request_id=request_id,
   858	                )
   859	            else:
   860	                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
   861	                    speaker=speaker,
   862	                    text=text,
   863	                    temperature=temperature,
   864	                    top_k=top_k,
   865	                    top_p=top_p,
   866	                    max_tokens=max_tokens,
   867	                    repetition_penalty=repetition_penalty,
   868	                    seed=seed,
   869	                    request_id=request_id,
   870	                )
   871
   872	            async for audio_chunk in stream_gen:
   873	                if first_chunk_time is None:
   874	                    first_chunk_time = time.time()
   875	                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
   876	                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
   877	                    audio_chunk = wav_header + audio_chunk
   878
   879	                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
   880	                metrics["chunks_sent"] += 1
   881	                metrics["total_bytes"] = total_pcm_bytes + 44
   882	                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
   883	                yield audio_chunk, metrics
   884	    finally:
   885	        active_requests = _with_inflight_delta(-1)
   886	        if _timing_enabled():
   887	            logger.info(
   888	                "tts_runtime_stream_end",
   889	                extra={
   890	                    "speaker": speaker,
   891	                    "request_id": request_id,
   892	                    "request_inflight": active_requests,
   893	                    "chunks_sent": metrics.get("chunks_sent", 0),
   894	                },
   895	            )
   896
   897
   898	async def _stream_chunked_text(
   899	    runtime: TTSRuntime,
   900	    long_processor,
   901	    text: str,
   902	    speaker: str,
   903	    temperature: float,
   904	    top_k: int,
   905	    top_p: float,
   906	    max_tokens: int,
   907	    repetition_penalty: float,
   908	    seed: Optional[int],
   909	    sample_rate: int,
   910	    request_id: Optional[str] = None,
   911	) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
   912	    """
   913	    Internal helper: stream chunked text with voice consistency.
   914
   915	    Uses global token caching from first chunk to maintain voice across chunks.
   916	    """
   917	    from veena3modal.audio.crossfade import crossfade_bytes_int16
   918
   919	    # Chunk text using the long text processor's chunker
   920	    chunks = long_processor.chunk_text(text)
   921
   922	    if not chunks:
   923	        return
   924
   925	    captured_globals: Optional[List[int]] = None
   926	    previous_chunk_tail: Optional[bytes] = None
   927	    chunk_metrics = {"chunks_processed": 0}
   928
   929	    for i, chunk_text in enumerate(chunks):
   930	        chunk_metrics["chunks_processed"] = i + 1
   931
   932	        if i == 0:
   933	            # First chunk: capture global tokens for voice consistency
   934	            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
   935	                speaker=speaker,
   936	                text=chunk_text,
   937	                temperature=temperature,
   938	                top_k=top_k,
   939	                top_p=top_p,
   940	                max_tokens=max_tokens,
   941	                repetition_penalty=repetition_penalty,
   942	                seed=seed,
   943	                request_id=request_id,
   944	            ):
   945	                if captured_globals is None and global_ids:
   946	                    captured_globals = global_ids
   947
   948	                # Crossfade with previous tail (inter-chunk stitching)
   949	                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   950	                    previous_chunk_tail,
   951	                    audio_bytes,
   952	                    sample_rate_hz=sample_rate,
   953	                    crossfade_ms=50,
   954	                )
   955
   956	                if to_emit:
   957	                    yield to_emit, chunk_metrics
   958	        else:
   959	            # Continuation chunks: use captured globals for voice consistency
   960	            if captured_globals is None:
   961	                logger.warning(f"No captured globals for chunk {i+1}, using regular streaming")
   962	                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
   963	                    speaker=speaker,
   964	                    text=chunk_text,
   965	                    temperature=temperature,
   966	                    top_k=top_k,
   967	                    top_p=top_p,
   968	                    max_tokens=max_tokens,
   969	                    repetition_penalty=repetition_penalty,
   970	                    seed=seed,
   971	                    request_id=request_id,
   972	                ):
   973	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   974	                        previous_chunk_tail,
   975	                        audio_bytes,
   976	                        sample_rate_hz=sample_rate,
   977	                        crossfade_ms=50,
   978	                    )
   979	                    if to_emit:
   980	                        yield to_emit, chunk_metrics


cd /home/ubuntu/veenaModal && nl -ba veena3modal/services/tts_runtime.py | sed -n '980,1130p'


Output:
Chunk ID: 10b680
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 332
Output:
   980	                        yield to_emit, chunk_metrics
   981	            else:
   982	                # Use continuation method with cached globals
   983	                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
   984	                    speaker=speaker,
   985	                    text=chunk_text,
   986	                    global_ids=captured_globals,
   987	                    temperature=temperature,
   988	                    top_k=top_k,
   989	                    top_p=top_p,
   990	                    max_tokens=max_tokens,
   991	                    repetition_penalty=repetition_penalty,
   992	                    seed=seed,
   993	                    request_id=request_id,
   994	                ):
   995	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   996	                        previous_chunk_tail,
   997	                        audio_bytes,
   998	                        sample_rate_hz=sample_rate,
   999	                        crossfade_ms=50,
  1000	                    )
  1001	                    if to_emit:
  1002	                        yield to_emit, chunk_metrics
  1003
  1004	    # Flush remaining tail
  1005	    if previous_chunk_tail:
  1006	        yield previous_chunk_tail, chunk_metrics


I’m implementing the benchmark-facing instrumentation first in the runtime/pipeline code paths, then I’ll add a dedicated local stress script that prints per-request GPU/LLM timings and concurrency summaries.


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py
@@
 import asyncio
 import os
 import re
-from statistics import median, mean
+import logging
 import torch
-from typing import AsyncGenerator, Optional, List
+from typing import AsyncGenerator, Optional, List, Dict, Any
 from vllm import SamplingParams
 from veena3modal.audio.crossfade import crossfade_bytes_int16
 from veena3modal.core.token_utils import BiCodecTokenParser

@@
 from veena3modal.core.constants import (
@@
 )
+
+logger = logging.getLogger(__name__)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


cd /home/ubuntu/veenaModal && sed -n '220,520p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 6f7ae6
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 3847
Output:
        global_ids = [int(t) for t in global_matches]

        return semantic_ids, global_ids

    async def generate_speech_stream_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,  # Added for Spark TTS parity with non-streaming
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        request_id: Optional[str] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech audio with sliding window streaming for Indic model.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "<laugh> Hello there!"
                - "नमस्ते! <excited> आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops
            seed: Random seed for reproducibility

        Yields:
            Audio bytes (int16 PCM, 16kHz mono - BiCodec)
        """
        import time
        t_start = time.time()
        emit_timing = _detailed_timing_enabled()

        # Build prompt using Indic prompt builder
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling (matching non-streaming pipeline parameters)
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID (128258)
        # which doesn't exist in the Spark vocab and would cause generation to run to max_tokens
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,  # Keep BiCodec tokens in output
            seed=seed,
        )

        # BiCodec token buffers (separate for semantic and global)
        semantic_buffer = []
        global_buffer = []
        processed_token_count = 0  # Track how many vLLM tokens we've parsed
        total_audio_chunks = 0

        # BiCodec streaming configuration
        EXPECTED_GLOBAL_COUNT = 32  # BiCodec uses 32 fixed global tokens (per paper)
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8, -120ms TTFB)
        # OPTIMIZATION: Increased from 24 to 48 to reduce O(n) re-decode calls by 2x.
        # At 50 TPS, 48 tokens = ~960ms audio per chunk. Cuts total decode calls roughly in half.
        DECODE_INTERVAL = 48
        CROSSFADE_MS = 50
        # OPTIMIZATION: Windowed decode - only decode last WINDOW_SIZE tokens instead of ALL.
        # BiCodec WaveGenerator has finite receptive field (~64 tokens based on kernel sizes).
        # Window must be >= receptive_field + DECODE_INTERVAL to avoid boundary artifacts.
        WINDOW_SIZE = 128  # 128 tokens covers receptive field + margin for crossfade

        # Generate tokens with vLLM (streaming)
        if request_id is None:
            import uuid
            request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser

        t_gen_start = time.time()
        batch_wall_ms: List[float] = []
        batch_token_counts: List[int] = []
        decode_calls = 0
        decode_wall_ms = 0.0
        decode_gpu_ms = 0.0
        parse_wall_ms = 0.0

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track timing
        t_first_token = None
        t_first_chunk_ready = None
        t_first_chunk_decoded = None
        iteration_count = 0
        last_decode_count = 0

        # Track audio samples EMITTED to the user (not just decoded)
        # BiCodec produces ~320 samples per semantic token at 16kHz
        # CRITICAL: This tracks samples that have been YIELDED, not held in tail
        total_samples_emitted_to_user = 0
        # Keep a tail to enable crossfading between consecutive emitted chunks
        previous_chunk_tail: Optional[bytes] = None

        # Stream tokens with BiCodec extraction
        async for request_output in results_generator:
            iteration_count += 1
            t_iter = time.time()

            # Track first token received
            if t_first_token is None:
                t_first_token = t_iter

            # Extract generated token IDs
            generated_ids = request_output.outputs[0].token_ids

            parse_start = time.time()
            # OPTIMIZATION: Only process NEW tokens (incremental parsing)
            # Old code: tokenizer.decode(ALL) + re.findall(ALL) = O(n²) per stream
            # New code: parse only new tokens = O(1) per token
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)

            # Parse new tokens and append to buffers (in-place modification)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
            parse_wall_ms += (time.time() - parse_start) * 1000
            batch_token_counts.append(len(new_token_ids))
            batch_wall_ms.append(0 if t_iter is None else (time.time() - t_iter) * 1000)

            # TRUE STREAMING with BiCodec two-phase generation:
            # Phase 1: Generate 32 global tokens (pre-roll)
            # Phase 2: Stream semantic tokens incrementally

            # Check if we have the 32 global tokens (pre-roll complete)
            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                # Pre-roll complete! Now we can stream semantic tokens
                if t_first_chunk_ready is None and len(semantic_buffer) >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    t_first_chunk_ready = time.time()

                # Stream semantic tokens incrementally
                # Only decode if we have enough NEW semantic tokens since last decode
                semantic_count = len(semantic_buffer)
                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    # Check if we should decode (every DECODE_INTERVAL new semantic tokens)
                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
                        last_decode_count = semantic_count

                        # OPTIMIZATION: Windowed decode instead of full re-decode
                        # Only decode the last WINDOW_SIZE tokens (O(1) per chunk instead of O(n))
                        # For first few chunks when total tokens < WINDOW_SIZE, decode all
                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]

                        if len(semantic_buffer) <= WINDOW_SIZE:
                            # Early phase: decode all tokens (window not yet relevant)
                            decode_semantic = semantic_buffer
                        else:
                            # Windowed: only decode last WINDOW_SIZE tokens
                            decode_semantic = semantic_buffer[-WINDOW_SIZE:]

                        t_decode_start = time.time()
                        if emit_timing and torch.cuda.is_available():
                            torch.cuda.synchronize()
                            decode_start_event = torch.cuda.Event(enable_timing=True)
                            decode_end_event = torch.cuda.Event(enable_timing=True)
                            decode_start_event.record()
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=decode_semantic,
                            global_ids=window_global,
                            trim_warmup=False,
                            use_sliding_window=False,
                        )
                        t_decode_end = time.time()
                        decode_calls += 1
                        decode_wall_ms += (t_decode_end - t_decode_start) * 1000
                        if emit_timing and torch.cuda.is_available():
                            decode_end_event.record()
                            torch.cuda.synchronize()
                            decode_gpu_ms += decode_start_event.elapsed_time(decode_end_event)

                        if audio_bytes:
                            total_samples_decoded = len(audio_bytes) // 2

                            if len(semantic_buffer) <= WINDOW_SIZE:
                                # Early phase: extract new samples from full decode
                                if total_samples_decoded > total_samples_emitted_to_user:
                                    new_bytes_start = total_samples_emitted_to_user * 2
                                    new_audio_bytes = audio_bytes[new_bytes_start:]
                                else:
                                    new_audio_bytes = b""
                            else:
                                # Windowed phase: take only the tail portion of the window decode
                                # The last DECODE_INTERVAL tokens' worth of audio is the new content
                                # Each token produces ~320 samples at 16kHz/50TPS
                                new_samples_approx = DECODE_INTERVAL * 320
                                if total_samples_decoded > new_samples_approx:
                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
                                else:
                                    new_bytes_start = 0
                                new_audio_bytes = audio_bytes[new_bytes_start:]

                            if new_audio_bytes:
                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                    previous_chunk_tail,
                                    new_audio_bytes,
                                    sample_rate_hz=16000,
                                    crossfade_ms=CROSSFADE_MS,
                                )

                                samples_emitted_in_this_chunk = len(to_emit) // 2
                                total_samples_emitted_to_user += samples_emitted_in_this_chunk

                                if to_emit:
                                    total_audio_chunks += 1
                                    if t_first_chunk_decoded is None:
                                        t_first_chunk_decoded = t_decode_end

                                    if emit_timing:
                                        logger.info(
                                            "tts_stream_decode_batch",
                                            extra={
                                                "speaker": speaker,
                                                "request_id": request_id,
                                                "batch_index": decode_calls,
                                                "decode_wall_ms": int((t_decode_end - t_decode_start) * 1000),
                                                "decode_gpu_ms": int(decode_start_event.elapsed_time(decode_end_event))
                                                if torch.cuda.is_available() else None,
                                                "semantic_count": len(semantic_buffer),
                                            },
                                        )

                                    yield to_emit

        # Final chunk: decode any remaining semantic tokens
        # Use the same 32 global tokens (they get pooled)
        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,  # All semantic tokens
                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],  # Use first 32 global tokens
                use_sliding_window=False,  # Get all samples
                trim_warmup=False
            )
            if audio_bytes:
                # Only yield NEW samples
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    samples_emitted_in_final = len(to_emit) // 2
                    total_samples_emitted_to_user += samples_emitted_in_final

                    if to_emit:
                        total_audio_chunks += 1
                        yield to_emit

        # Flush any remaining tail that was held back for crossfade
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            yield previous_chunk_tail

        # Log completion summary
        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000  # 16kHz sample rate
        ttfb_ms = (t_first_chunk_decoded - t_start) * 1000 if t_first_chunk_decoded else 0
        print(f"🎵 Streaming complete: {audio_duration_s:.2f}s audio, TTFB: {ttfb_ms:.0f}ms, {total_audio_chunks} chunks")

    async def generate_speech_stream_indic_first_chunk(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[tuple, None]:
        """
        Generate speech for the FIRST chunk of multi-chunk text, capturing global tokens.

        This method is specifically for chunked generation: it yields (audio_bytes, global_ids)
        tuples instead of just audio_bytes. The caller captures global_ids from the first


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py
@@
     async def generate_speech_stream_indic(
         self,
         speaker: str,
         text: str,
         temperature: float = DEFAULT_TEMPERATURE,
         top_k: int = DEFAULT_TOP_K,  # Added for Spark TTS parity with non-streaming
         top_p: float = DEFAULT_TOP_P,
         max_tokens: int = DEFAULT_MAX_TOKENS,
         repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
         seed: Optional[int] = None,
         request_id: Optional[str] = None,
+        request_metrics: Optional[Dict[str, Any]] = None,
     ) -> AsyncGenerator[bytes, None]:
@@
-        import time
-        t_start = time.time()
-        emit_timing = _detailed_timing_enabled()
+        import statistics
+        import time
+        t_start = time.time()
+        emit_timing = _detailed_timing_enabled()
+        request_metrics = request_metrics or {}
+        request_metrics.update(
+            {
+                "request_id": request_id,
+                "speaker": speaker,
+                "text_length": len(text),
+            }
+        )
@@
         if request_id is None:
             import uuid
             request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
+            request_metrics["request_id"] = request_id

         # Reuse pre-warmed token parser (singleton, created once in __init__)
         token_parser = self.token_parser
+        if token_parser is None:
+            raise RuntimeError("BiCodecTokenParser unavailable for streaming")

         t_gen_start = time.time()
         batch_wall_ms: List[float] = []
         batch_token_counts: List[int] = []
+        batch_gpu_ms: List[float] = []
+        parse_wall_ms_list: List[float] = []
         decode_calls = 0
         decode_wall_ms = 0.0
         decode_gpu_ms = 0.0
-        parse_wall_ms = 0.0
+        decode_wall_samples: List[float] = []
+        decode_gpu_samples: List[float] = []

         results_generator = self.model.engine.generate(
             prompt=prompt,
             sampling_params=sampling_params,
             request_id=request_id,
@@
         # Stream tokens with BiCodec extraction
         async for request_output in results_generator:
             iteration_count += 1
             t_iter = time.time()
@@
             new_token_ids = generated_ids[processed_token_count:]
             processed_token_count = len(generated_ids)

             # Parse new tokens and append to buffers (in-place modification)
             token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
-            parse_wall_ms += (time.time() - parse_start) * 1000
+            parse_wall_ms = (time.time() - parse_start) * 1000
+            parse_wall_ms_list.append(parse_wall_ms)
             batch_token_counts.append(len(new_token_ids))
-            batch_wall_ms.append(0 if t_iter is None else (time.time() - t_iter) * 1000)
+            batch_wall_ms.append((time.time() - t_iter) * 1000 if t_iter else 0.0)
@@
-                        t_decode_start = time.time()
+                        decode_gpu_ms_this = None
+                        t_decode_start = time.time()
                         if emit_timing and torch.cuda.is_available():
                             torch.cuda.synchronize()
                             decode_start_event = torch.cuda.Event(enable_timing=True)
                             decode_end_event = torch.cuda.Event(enable_timing=True)
                             decode_start_event.record()
@@
                         )
                         t_decode_end = time.time()
                         decode_calls += 1
-                        decode_wall_ms += (t_decode_end - t_decode_start) * 1000
+                        decode_ms = (t_decode_end - t_decode_start) * 1000
+                        decode_wall_ms += decode_ms
+                        decode_wall_samples.append(decode_ms)
                         if emit_timing and torch.cuda.is_available():
                             decode_end_event.record()
                             torch.cuda.synchronize()
-                            decode_gpu_ms += decode_start_event.elapsed_time(decode_end_event)
+                            decode_gpu_ms_this = decode_start_event.elapsed_time(decode_end_event)
+                            decode_gpu_ms += decode_gpu_ms_this
+                            decode_gpu_samples.append(decode_gpu_ms_this)
+                            batch_gpu_ms.append(decode_gpu_ms_this)
@@
                         samples_emitted_in_this_chunk = len(to_emit) // 2
                         total_samples_emitted_to_user += samples_emitted_in_this_chunk
@@
                                     logger.info(
                                         "tts_stream_decode_batch",
                                         extra={
                                             "speaker": speaker,
                                             "request_id": request_id,
                                             "batch_index": decode_calls,
                                             "decode_wall_ms": int((t_decode_end - t_decode_start) * 1000),
-                                            "decode_gpu_ms": int(decode_start_event.elapsed_time(decode_end_event))
-                                            if torch.cuda.is_available() else None,
+                                            "decode_gpu_ms": int(decode_gpu_ms_this)
+                                            if decode_gpu_ms_this is not None else None,
                                             "semantic_count": len(semantic_buffer),
                                         },
                                     )
@@
-            if audio_bytes:
+            if audio_bytes:
                 # Only yield NEW samples
                 total_samples_decoded = len(audio_bytes) // 2

                 if total_samples_decoded > total_samples_emitted_to_user:
                     new_bytes_start = total_samples_emitted_to_user * 2
                     new_audio_bytes = audio_bytes[new_bytes_start:]
@@
                     if to_emit:
                         total_audio_chunks += 1
                         yield to_emit

         # Flush any remaining tail that was held back for crossfade
         if previous_chunk_tail:
             total_audio_chunks += 1
             tail_samples = len(previous_chunk_tail) // 2
             total_samples_emitted_to_user += tail_samples
             yield previous_chunk_tail

         # Log completion summary
         t_end = time.time()
         audio_duration_s = total_samples_emitted_to_user / 16000  # 16kHz sample rate
-        ttfb_ms = (t_first_chunk_decoded - t_start) * 1000 if t_first_chunk_decoded else 0
-        print(f"🎵 Streaming complete: {audio_duration_s:.2f}s audio, TTFB: {ttfb_ms:.0f}ms, {total_audio_chunks} chunks")
+        ttfb_ms = (
+            (t_first_chunk_decoded - t_start) * 1000
+            if t_first_chunk_decoded is not None
+            else (t_first_token - t_start) * 1000
+            if t_first_token is not None
+            else 0
+        )
+        total_wall_ms = (t_end - t_start) * 1000
+        request_metrics.update(
+            {
+                "llm_request_id": request_id,
+                "generated_tokens": len(semantic_buffer) + len(global_buffer),
+                "semantic_tokens": len(semantic_buffer),
+                "global_tokens": len(global_buffer),
+                "llm_batch_count": len(batch_wall_ms),
+                "llm_batch_tokens": batch_token_counts,
+                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
+                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
+                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
+                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
+                "llm_parse_ms": int(sum(parse_wall_ms_list)),
+                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
+                "decode_calls": decode_calls,
+                "llm_decode_wall_ms_total": int(decode_wall_ms),
+                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
+                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
+                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
+                "ttfb_ms": int(ttfb_ms),
+                "generation_ms": int(total_wall_ms),
+                "audio_duration_seconds": audio_duration_s,
+                "audio_chunks": total_audio_chunks,
+                "samples_emitted": total_samples_emitted_to_user,
+                "text_chunked": False,
+                "concurrency_mode": "single",
+            }
+        )
+        if emit_timing:
+            logger.info("tts_stream_indic_complete", extra=request_metrics)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py
@@
     async def generate_speech_stream_indic_first_chunk(
         self,
         speaker: str,
         text: str,
         temperature: float = DEFAULT_TEMPERATURE,
         top_k: int = DEFAULT_TOP_K,
         top_p: float = DEFAULT_TOP_P,
         max_tokens: int = DEFAULT_MAX_TOKENS,
         repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
         seed: Optional[int] = None,
         request_id: Optional[str] = None,
+        request_metrics: Optional[Dict[str, Any]] = None,
     ) -> AsyncGenerator[tuple, None]:
@@
         import time
+        request_metrics = request_metrics or {}
         t_start = time.time()
@@
         import uuid
         request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"

         # Reuse pre-warmed token parser (singleton, created once in __init__)
         token_parser = self.token_parser
+        if token_parser is None:
+            raise RuntimeError("BiCodecTokenParser unavailable for first-chunk streaming")
@@
         if previous_chunk_tail:
             total_audio_chunks += 1
             tail_samples = len(previous_chunk_tail) // 2
             total_samples_emitted_to_user += tail_samples
             yield (previous_chunk_tail, captured_globals)

         t_end = time.time()
         audio_duration_s = total_samples_emitted_to_user / 16000
-        print(f"🎵 First chunk complete: {audio_duration_s:.2f}s audio, {total_audio_chunks} chunks, captured {len(captured_globals)} globals")
+        request_metrics.update(
+            {
+                "request_id": request_id,
+                "audio_duration_seconds": audio_duration_s,
+                "audio_chunks": total_audio_chunks,
+                "text_chunked": True,
+                "captured_globals": len(captured_globals),
+                "global_tokens": len(captured_globals),
+                "semantic_tokens": len(semantic_buffer),
+                "generation_ms": int((t_end - t_start) * 1000),
+                "ttfb_ms": int((t_end - t_start) * 1000) if captured_globals else 0,
+            }
+        )
+        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)
*** End Patch


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:
    async def generate_speech_stream_indic_first_chunk(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        request_id: Optional[str] = None,
    ) -> AsyncGenerator[tuple, None]:


cd /home/ubuntu/veenaModal && sed -n '500,760p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 32c5d7
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 3157
Output:

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    samples_emitted_in_final = len(to_emit) // 2
                    total_samples_emitted_to_user += samples_emitted_in_final

                    if to_emit:
                        total_audio_chunks += 1
                        yield to_emit

        # Flush any remaining tail that was held back for crossfade
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            yield previous_chunk_tail

        # Log completion summary
        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000  # 16kHz sample rate
        ttfb_ms = (
            (t_first_chunk_decoded - t_start) * 1000
            if t_first_chunk_decoded is not None
            else (t_first_token - t_start) * 1000
            if t_first_token is not None
            else 0
        )
        total_wall_ms = (t_end - t_start) * 1000
        request_metrics.update(
            {
                "llm_request_id": request_id,
                "generated_tokens": len(semantic_buffer) + len(global_buffer),
                "semantic_tokens": len(semantic_buffer),
                "global_tokens": len(global_buffer),
                "llm_batch_count": len(batch_wall_ms),
                "llm_batch_tokens": batch_token_counts,
                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_parse_ms": int(sum(parse_wall_ms_list)),
                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
                "decode_calls": decode_calls,
                "llm_decode_wall_ms_total": int(decode_wall_ms),
                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
                "ttfb_ms": int(ttfb_ms),
                "generation_ms": int(total_wall_ms),
                "audio_duration_seconds": audio_duration_s,
                "audio_chunks": total_audio_chunks,
                "samples_emitted": total_samples_emitted_to_user,
                "text_chunked": False,
                "concurrency_mode": "single",
            }
        )
        if emit_timing:
            logger.info("tts_stream_indic_complete", extra=request_metrics)

    async def generate_speech_stream_indic_first_chunk(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[tuple, None]:
        """
        Generate speech for the FIRST chunk of multi-chunk text, capturing global tokens.

        This method is specifically for chunked generation: it yields (audio_bytes, global_ids)
        tuples instead of just audio_bytes. The caller captures global_ids from the first
        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.

        Use Case (chunked streaming with voice consistency):
            globals_captured = None
            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
                if globals_captured is None and global_ids:
                    globals_captured = global_ids  # Capture once
                yield audio_bytes
            # Now use globals_captured for subsequent chunks

        Args:
            speaker: Speaker name
            text: First text chunk to synthesize
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max tokens to generate
            repetition_penalty: Prevent repetition
            seed: Random seed for reproducibility

        Yields:
            Tuple of (audio_bytes, global_ids)
            - audio_bytes: Raw PCM audio (int16, 16kHz)
            - global_ids: List of 32 global token IDs (populated after first decode, else empty)

        Thread Safety:
            This method is stateless and thread-safe. Each call creates its own
            request-scoped state. No global state is modified or shared.
        """
        import time
        t_start = time.time()

        # Build prompt using Indic prompt builder
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,
            seed=seed,
        )

        # BiCodec token buffers
        semantic_buffer = []
        global_buffer = []
        processed_token_count = 0  # Track how many vLLM tokens we've parsed
        total_audio_chunks = 0

        # BiCodec streaming configuration
        EXPECTED_GLOBAL_COUNT = 32
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
        CROSSFADE_MS = 50

        # Generate unique request ID
        import uuid
        request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None
        captured_globals: List[int] = []  # Will be populated once we have 32 globals

        # Stream tokens with BiCodec extraction
        async for request_output in results_generator:
            generated_ids = request_output.outputs[0].token_ids

            # OPTIMIZATION: Incremental parsing (O(1) per new token)
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)

            # Capture global tokens once we have all 32
            if not captured_globals and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                captured_globals = global_buffer[:EXPECTED_GLOBAL_COUNT]

            # Process if we have all global tokens
            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                semantic_count = len(semantic_buffer)
                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
                        last_decode_count = semantic_count

                        all_semantic = semantic_buffer
                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]

                        if self.snac_decoder.enable_batching:
                            audio_bytes = await self.snac_decoder.decode_single_async(
                                semantic_ids=all_semantic,
                                global_ids=window_global,
                                trim_warmup=False,
                                use_sliding_window=False
                            )
                        else:
                            audio_bytes = self.snac_decoder.decode_streaming(
                                semantic_ids=all_semantic,
                                global_ids=window_global,
                                use_sliding_window=False,
                                trim_warmup=False
                            )

                        if audio_bytes:
                            total_samples_decoded = len(audio_bytes) // 2

                            if total_samples_decoded > total_samples_emitted_to_user:
                                new_bytes_start = total_samples_emitted_to_user * 2
                                new_audio_bytes = audio_bytes[new_bytes_start:]

                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                    previous_chunk_tail,
                                    new_audio_bytes,
                                    sample_rate_hz=16000,
                                    crossfade_ms=CROSSFADE_MS,
                                )

                                samples_emitted_in_this_chunk = len(to_emit) // 2
                                total_samples_emitted_to_user += samples_emitted_in_this_chunk

                                if to_emit:
                                    total_audio_chunks += 1
                                    # Yield tuple: (audio_bytes, global_ids)
                                    yield (to_emit, captured_globals)

        # Final chunk processing
        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,
                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
                use_sliding_window=False,
                trim_warmup=False
            )
            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    samples_emitted_in_final = len(to_emit) // 2
                    total_samples_emitted_to_user += samples_emitted_in_final

                    if to_emit:
                        total_audio_chunks += 1
                        yield (to_emit, captured_globals)

        # Flush remaining tail
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples


cd /home/ubuntu/veenaModal && sed -n '760,900p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 11e71b
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1581
Output:
            total_samples_emitted_to_user += tail_samples
            yield (previous_chunk_tail, captured_globals)

        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000
        print(f"🎵 First chunk complete: {audio_duration_s:.2f}s audio, {total_audio_chunks} chunks, captured {len(captured_globals)} globals")

    async def generate_speech_stream_indic_continuation(
        self,
        speaker: str,
        text: str,
        global_ids: List[int],
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech for CONTINUATION chunks using pre-captured global tokens.

        This is the key method for voice consistency in chunked generation:
        - Uses global_ids from the first chunk to maintain identical voice
        - Model generates only semantic tokens (skips global token generation)
        - Ensures no voice drift across chunks

        CRITICAL for production:
        - global_ids is request-scoped, passed explicitly (no shared state)
        - Thread-safe: each request has its own state
        - No global caching that could cause cross-request contamination

        Args:
            speaker: Speaker name (must match first chunk)
            text: Continuation text chunk to synthesize
            global_ids: 32 global token IDs captured from first chunk
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max tokens to generate
            repetition_penalty: Prevent repetition
            seed: Random seed (use same as first chunk for consistency)

        Yields:
            Audio bytes (int16 PCM, 16kHz mono)

        Raises:
            ValueError: If global_ids doesn't contain exactly 32 tokens

        Thread Safety:
            Fully thread-safe. All state is request-scoped and passed explicitly.
        """
        import time
        t_start = time.time()

        # Validate global tokens
        EXPECTED_GLOBAL_COUNT = 32
        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
            raise ValueError(
                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
                f"global_ids must be captured from first chunk generation."
            )

        # Build prompt WITH pre-filled global tokens
        # This tells the model to skip global generation and go straight to semantic tokens
        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)

        # Configure sampling
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,
            seed=seed,
        )

        # BiCodec token buffers
        semantic_buffer = []
        global_buffer_unused = []  # Not used in continuation, but needed for parser API
        processed_token_count = 0  # Track how many vLLM tokens we've parsed
        total_audio_chunks = 0

        # BiCodec streaming configuration
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
        CROSSFADE_MS = 50

        # Generate unique request ID
        import uuid
        request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None

        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
        async for request_output in results_generator:
            generated_ids = request_output.outputs[0].token_ids

            # OPTIMIZATION: Incremental parsing (O(1) per new token)
            # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)

            semantic_count = len(semantic_buffer)
            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                if semantic_count - last_decode_count >= DECODE_INTERVAL:
                    last_decode_count = semantic_count

                    # Use pre-captured global tokens for decoding
                    if self.snac_decoder.enable_batching:
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=semantic_buffer,
                            global_ids=global_ids,  # Use captured globals
                            trim_warmup=False,
                            use_sliding_window=False
                        )
                    else:
                        audio_bytes = self.snac_decoder.decode_streaming(
                            semantic_ids=semantic_buffer,
                            global_ids=global_ids,  # Use captured globals
                            use_sliding_window=False,
                            trim_warmup=False
                        )


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py
@@
-    async def generate_speech_stream_indic_first_chunk(
-        self,
-        speaker: str,
-        text: str,
-        temperature: float = DEFAULT_TEMPERATURE,
-        top_k: int = DEFAULT_TOP_K,
-        top_p: float = DEFAULT_TOP_P,
-        max_tokens: int = DEFAULT_MAX_TOKENS,
-        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
-        seed: Optional[int] = None,
-    ) -> AsyncGenerator[tuple, None]:
+    async def generate_speech_stream_indic_first_chunk(
+        self,
+        speaker: str,
+        text: str,
+        temperature: float = DEFAULT_TEMPERATURE,
+        top_k: int = DEFAULT_TOP_K,
+        top_p: float = DEFAULT_TOP_P,
+        max_tokens: int = DEFAULT_MAX_TOKENS,
+        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
+        seed: Optional[int] = None,
+        request_id: Optional[str] = None,
+        request_metrics: Optional[Dict[str, Any]] = None,
+    ) -> AsyncGenerator[tuple, None]:
@@
-        t_start = time.time()
+        import statistics
+        t_start = time.time()
+        request_metrics = request_metrics or {}
+        request_metrics.setdefault("request_id", request_id)
@@
-        import uuid
-        request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
+        import uuid
+        if request_id is None:
+            request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
+            request_metrics["request_id"] = request_id
@@
-        # Reuse pre-warmed token parser (singleton, created once in __init__)
+        # Reuse pre-warmed token parser (singleton, created once in __init__)
         token_parser = self.token_parser
+        if token_parser is None:
+            raise RuntimeError("BiCodecTokenParser unavailable for first-chunk streaming")
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py
@@
         if previous_chunk_tail:
             total_audio_chunks += 1
             tail_samples = len(previous_chunk_tail) // 2
             total_samples_emitted_to_user += tail_samples
             yield (previous_chunk_tail, captured_globals)

         t_end = time.time()
         audio_duration_s = total_samples_emitted_to_user / 16000
-        print(f"🎵 First chunk complete: {audio_duration_s:.2f}s audio, {total_audio_chunks} chunks, captured {len(captured_globals)} globals")
+        request_metrics.update(
+            {
+                "request_id": request_id,
+                "speaker": speaker,
+                "audio_duration_seconds": audio_duration_s,
+                "audio_chunks": total_audio_chunks,
+                "text_chunked": True,
+                "captured_global_count": len(captured_globals),
+                "semantic_tokens": len(semantic_buffer),
+                "generation_ms": int((t_end - t_start) * 1000),
+            }
+        )
+        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py
@@
     async def generate_speech_stream_indic_continuation(
         self,
         speaker: str,
         text: str,
         global_ids: List[int],
         temperature: float = DEFAULT_TEMPERATURE,
         top_k: int = DEFAULT_TOP_K,
         top_p: float = DEFAULT_TOP_P,
         max_tokens: int = DEFAULT_MAX_TOKENS,
         repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
         seed: Optional[int] = None,
+        request_id: Optional[str] = None,
+        request_metrics: Optional[Dict[str, Any]] = None,
     ) -> AsyncGenerator[bytes, None]:
@@
-        import time
+        import statistics
+        import time
         t_start = time.time()
+        request_metrics = request_metrics or {}
+        request_metrics.setdefault("request_id", request_id)
@@
-        import uuid
-        request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
+        import uuid
+        if request_id is None:
+            request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
+            request_metrics["request_id"] = request_id
@@
         # Reuse pre-warmed token parser (singleton, created once in __init__)
         token_parser = self.token_parser
+        if token_parser is None:
+            raise RuntimeError("BiCodecTokenParser unavailable for continuation streaming")
@@
         # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
         async for request_output in results_generator:
@@
                     if to_emit:
                         total_audio_chunks += 1
                         yield to_emit
+
+        t_end = time.time()
+        request_metrics.update(
+            {
+                "request_id": request_id,
+                "speaker": speaker,
+                "audio_duration_seconds": total_samples_emitted_to_user / 16000,
+                "audio_chunks": total_audio_chunks,
+                "semantic_tokens": len(semantic_buffer),
+                "global_tokens": len(global_ids),
+                "generation_ms": int((t_end - t_start) * 1000),
+            }
+        )
+        logger.info("tts_stream_continuation_complete", extra=request_metrics)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


cd /home/ubuntu/veenaModal && rg -n "statistics" veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 49b5e3
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 106
Output:
258:        import statistics
544:                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
548:                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
555:                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
619:        import statistics
835:        import statistics


cd /home/ubuntu/veenaModal && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '780,930p'