Output:
Chunk ID: 07bdd9
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 26778
Output:
Total output lines: 2304

     1	"""
     2	TTS runtime wrapper (container-scoped singleton).
     3
     4	In Modal, each container should load the model once and then handle many requests.
     5	This wrapper owns:
     6	- SparkTTSModel (vLLM engine)
     7	- IndicPromptBuilder
     8	- BiCodecDecoder
     9	- SparkTTSPipeline + Veena3SlidingWindowPipeline
    10	- optional SuperResolutionService
    11
    12	Imports framework-agnostic inference + processing code from `veena3modal/core`,
    13	`veena3modal/processing`, and `veena3modal/audio`.
    14	"""
    15
    16	from __future__ import annotations
    17
    18	import os
    19	import sys
    20	import time
    21	import logging
    22	from threading import Lock
    23	from dataclasses import dataclass, field
    24	from typing import Any, Dict, List, Optional, Tuple
    25
    26	logger = logging.getLogger(__name__)
    27
    28	_PROFILE_REQUEST_TIMING = "VEENA3_TIMING_LOGS"
    29	_inflight_counter = 0
    30	_inflight_lock = Lock()
    31
    32
    33	def _timing_enabled() -> bool:
    34	    """Return True when detailed request timing should be emitted."""
    35	    return os.getenv(_PROFILE_REQUEST_TIMING, "").lower() in {"1", "true", "yes", "on"}
    36
    37
    38	def _with_inflight_delta(delta: int) -> int:
    39	    """Update in-flight request counter and return the new value."""
    40	    global _inflight_counter
    41	    with _inflight_lock:
    42	        _inflight_counter = max(0, _inflight_counter + delta)
    43	        return _inflight_counter
    44
    45
    46	@dataclass
    47	class TTSRuntime:
    48	    """
    49	    Holds long-lived, per-container inference objects.
    50
    51	    Thread Safety:
    52	        This class is designed for async concurrency within a single container.
    53	        The vLLM engine handles internal batching and scheduling.
    54	        Do NOT share across processes without proper synchronization.
    55	    """
    56	    model: Any = None
    57	    pipeline: Any = None
    58	    streaming_pipeline: Any = None
    59	    prompt_builder: Any = None
    60	    bicodec_decoder: Any = None
    61	    sr_service: Optional[Any] = None
    62
    63	    model_version: str = "not_loaded"
    64	    is_loaded: bool = False
    65	    load_time_ms: float = 0.0
    66
    67	    # Configuration
    68	    model_path: str = ""
    69	    bicodec_path: str = ""
    70	    sr_checkpoint_dir: Optional[str] = None
    71	    device: str = "cuda"
    72
    73	    # OPTIMIZATION: Pre-computed global tokens per speaker (eliminates ~110ms pre-roll in streaming)
    74	    # Maps speaker_name -> list of 32 global token IDs captured at startup
    75	    speaker_global_cache: Dict[str, List[int]] = field(default_factory=dict)
    76
    77
    78	# Module-level singleton (per-container)
    79	_runtime: Optional[TTSRuntime] = None
    80
    81
    82	def get_runtime() -> Optional[TTSRuntime]:
    83	    """Get the current TTS runtime singleton."""
    84	    return _runtime
    85
    86
    87	def is_initialized() -> bool:
    88	    """Check if the runtime is initialized and ready."""
    89	    return _runtime is not None and _runtime.is_loaded
    90
    91
    92	def initialize_runtime(
    93	    model_path: Optional[str] = None,
    94	    bicodec_path: Optional[str] = None,
    95	    sr_checkpoint_dir: Optional[str] = None,
    96	    device: str = "cuda",
    97	    hf_token: Optional[str] = None,
    98	    gpu_memory_utilization: float = 0.25,
    99	    enable_sr: bool = False,
   100	    num_engines: int = 1,
   101	) -> TTSRuntime:
   102	    """
   103	    Initialize the TTS runtime with all components.
   104
   105	    This should be called once per container (e.g., in Modal's @modal.enter).
   106
   107	    Args:
   108	        model_path: Path to Spark TTS model (env: SPARK_TTS_MODEL_PATH)
   109	        bicodec_path: Path to BiCodec model (env: BICODEC_MODEL_PATH, defaults to model_path)
   110	        sr_checkpoint_dir: Path to super-resolution checkpoints (env: AP_BWE_CHECKPOINT_DIR)
   111	        device: Device for inference (cuda/cpu)
   112	        hf_token: HuggingFace token for private models (env: HF_TOKEN)
   113	        gpu_memory_utilization: vLLM GPU memory fraction (default: 0.25)
   114	        enable_sr: Enable super-resolution service
   115	        num_engines: Number of vLLM engine instances (default: 1, set to 2-3 for Tier 3 optimization)
   116
   117	    Returns:
   118	        Initialized TTSRuntime instance
   119
   120	    Raises:
   121	        RuntimeError: If initialization fails
   122	    """
   123	    global _runtime
   124
   125	    start_time = time.time()
   126
   127	    # Resolve paths from env vars if not provided
   128	    model_path = model_path or os.environ.get(
   129	        'SPARK_TTS_MODEL_PATH',
   130	        os.environ.get('MODEL_PATH', '/models/spark_tts_4speaker')
   131	    )
   132	    bicodec_path = bicodec_path or os.environ.get(
   133	        'BICODEC_MODEL_PATH',
   134	        model_path  # BiCodec is usually in the same directory
   135	    )
   136	    sr_checkpoint_dir = sr_checkpoint_dir or os.environ.get('AP_BWE_CHECKPOINT_DIR')
   137	    hf_token = hf_token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')
   138
   139	    logger.info(f"Initializing TTS runtime...")
   140	    logger.info(f"  Model path: {model_path}")
   141	    logger.info(f"  BiCodec path: {bicodec_path}")
   142	    logger.info(f"  SR enabled: {enable_sr}, path: {sr_checkpoint_dir}")
   143
   144	    try:
   145	        # Import inference components (vendored into veena3modal for Modal deployment)
   146	        from veena3modal.core.model_loader import SparkTTSModel
   147	        from veena3modal.core.pipeline import SparkTTSPipeline
   148	        from veena3modal.core.bicodec_decoder import BiCodecDecoder
   149	        from veena3modal.processing.prompt_builder import IndicPromptBuilder
   150
   151	        # Load SparkTTS model with vLLM engine
   152	        if num_engines > 1:
   153	            # TIER 3 OPTIMIZATION: Multiple vLLM engines on same GPU
   154	            # Each engine gets gpu_memory_utilization / num_engines to share GPU fairly.
   155	            # E.g., 3 engines at 0.08 each = 0.24 total, ~8GB per engine, ~24GB total.
   156	            from veena3modal.core.multi_engine import create_multi_engine_model
   157	            per_engine_mem = gpu_memory_utilization / num_engines
   158	            logger.info(f"Loading {num_engines} vLLM engines ({per_engine_mem:.2f} GPU mem each)...")
   159	            model = create_multi_engine_model(
   160	                model_path=model_path,
   161	                num_engines=num_engines,
   162	                hf_token=hf_token,
   163	                gpu_memory_per_engine=per_engine_mem,
   164	            )
   165	        else:
   166	            logger.info("Loading SparkTTS model with vLLM...")
   167	            model = SparkTTSModel(
   168	                model_path=model_path,
   169	                hf_token=hf_token,
   170	                gpu_memory_utilization=gpu_memory_utilization,
   171	            )
   172
   173	        # Initialize prompt builder
   174	        logger.info("Initializing prompt builder...")
   175	        prompt_builder = IndicPromptBuilder(
   176	            tokenizer=model.tokenizer,
   177	            model=model,
   178	        )
   179
   180	        # Initialize BiCodec decoder
   181	        logger.info("Initializing BiCodec decoder...")
   182	        bicodec_decoder = BiCodecDecoder(
   183	            device=device,
   184	            model_path=bicodec_path,
   185	        )
   186
   187	        # Initialize pipeline
   188	        logger.info("Initializing TTS pipeline...")
   189	        pipeline = SparkTTSPipeline(
   190	            model=model,
   191	            prompt_builder=prompt_builder,
   192	            bicodec_decoder=bicodec_decoder,
   193	        )
   194
   195	        # Initialize streaming pipeline (for M4)
   196	        streaming_pipeline = None
   197	        try:
   198	            from veena3modal.core.streaming_pipeline import Veena3SlidingWindowPipeline
   199	            logger.info("Initializing streaming pipeline...")
   200	            # NOTE: Parameter is named 'snac_decoder' for legacy reasons, but works with BiCodecDecoder
   201	            streaming_pipeline = Veena3SlidingWindowPipeline(
   202	                model=model,
   203	                prompt_builder=prompt_builder,
   204	                snac_decoder=bicodec_decoder,  # BiCodecDecoder is interface-compatible
   205	            )
   206	        except ImportError as e:
   207	            logger.warning(f"Streaming pipeline not available: {e}")
   208
   209	        # Initialize super-resolution (optional)
   210	        sr_service = None
   211	        if enable_sr and sr_checkpoint_dir:
   212	            try:
   213	                from veena3modal.core.super_resolution import SuperResolutionService
   214	                logger.info(f"Initializing super-resolution from {sr_checkpoint_dir}...")
   215	                sr_service = SuperResolutionService(checkpoint_dir=sr_checkpoint_dir)
   216	                # Load the model explicitly
   217	                if sr_service.load_model(device=device):
   218	                    logger.info("✅ Super-resolution model loaded successfully")
   219	                else:
   220	                    logger.warning("Super-resolution model failed to load")
   221	                    sr_service = None
   222	            except Exception as e:
   223	                logger.warning(f"Super-resolution not available: {e}")
   224
   225	        # Determine model version
   226	        model_version = os.path.basename(model_path.rstrip('/'))
   227	        if not model_version:
   228	            model_version = "spark-tts"
   229
   230	        load_time = (time.time() - start_time) * 1000
   231
   232	        # Create runtime
   233	        _runtime = TTSRuntime(
   234	            model=model,
   235	            pipeline=pipeline,
   236	            streaming_pipeline=streaming_pipeline,
   237	            prompt_builder=prompt_builder,
   238	            bicodec_decoder=bicodec_decoder,
   239	            sr_service=sr_service,
   240	            model_version=model_version,
   241	            is_loaded=True,
   242	            load_time_ms=load_time,
   243	            model_path=model_path,
   244	            bicodec_path=bicodec_path,
   245	            sr_checkpoint_dir=sr_checkpoint_dir,
   246	            device=device,
   247	        )
   248
   249	        logger.info(f"✅ TTS runtime initialized in {load_time:.0f}ms")
   250	        logger.info(f"   Model version: {model_version}")
   251
   252	        # OPTIMIZATION: Pre-compute global tokens for all speakers
   253	        # Disabled with multi-engine to avoid async event loop conflicts
   254	        if num_engines <= 1:
   255	            _precompute_speaker_globals(_runtime)
   256	        else:
   257	            logger.info("Speaker globals cache skipped (multi-engine mode)")
   258
   259	        # Update FastAPI app with model version
   260	        try:
   260	        try:
   261	            from veena3modal.api.fastapi_app import set_model_version
   262	            set_model_version(model_version)
   263	        except ImportError:
   264	            pass
   265
   266	        return _runtime
   267
   268	    except Exception as e:
   269	        logger.error(f"❌ Failed to initialize TTS runtime: {e}")
   270	        import traceback
   271	        traceback.print_exc()
   272	        raise RuntimeError(f"TTS runtime initialization failed: {e}") from e
   273
   274
   275	def _precompute_speaker_globals(runtime: TTSRuntime) -> None:
   276	    """
   277	    Pre-compute and cache global tokens for all 12 speakers at startup.
   278
   279	    OPTIMIZATION: BiCodec streaming requires 32 "global tokens" before any audio can be emitted.
   280	    These encode speaker identity via FSQ quantization. Since we have only 12 fixed speakers,
   281	    we generate one short utterance per speaker at startup, capture the 32 global tokens,
   282	    and cache them. During streaming, we inject cached globals via build_prefix_with_globals(),
   283	    skipping the ~110ms global token pre-roll phase entirely.
   284
   285	    This runs synchronously at startup (adds ~5-10s to cold start, saves ~110ms per streaming TTFB).
   286	    """
   287	    import asyncio
   288	    import re
   289	    from vllm import SamplingParams
   290	    from veena3modal.core.constants import (
   291	        SPEAKER_MAP, TRAINING_STOP_TOKEN_IDS,
   292	        DEFAULT_TEMPERATURE, DEFAULT_TOP_K, DEFAULT_TOP_P,
   293	    )
   294
   295	    logger.info("Pre-computing global tokens for all speakers...")
   296	    t_start = time.time()
   297
   298	    test_text = "Hello, this is a voice test."
   299
   300	    sampling_params = SamplingParams(
   301	        temperature=DEFAULT_TEMPERATURE,
   302	        top_k=DEFAULT_TOP_K,
   303	        top_p=DEFAULT_TOP_P,
   304	        max_tokens=128,  # Only need ~60 tokens (32 global + some semantic)
   305	        stop=TRAINING_STOP_TOKEN_IDS,
   306	        skip_special_tokens=False,
   307	    )
   308
   309	    async def _capture_globals_for_speaker(speaker_name: str) -> Optional[List[int]]:
   310	        """Generate a short utterance and capture the 32 global tokens."""
   311	        prompt = runtime.prompt_builder.build_prefix(speaker_name, test_text)
   312	        request_id = f"warmup-{speaker_name}-{int(time.time() * 1000)}"
   313
   314	        final_output = None
   315	        async for request_output in runtime.model.engine.generate(
   316	            prompt=prompt,
   317	            sampling_params=sampling_params,
   318	            request_id=request_id,
   319	        ):
   320	            final_output = request_output
   321
   322	        if final_output is None:
   323	            return None
   324
   325	        generated_text = final_output.outputs[0].text
   326	        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
   327	        global_ids = [int(t) for t in global_matches]
   328
   329	        if len(global_ids) >= 32:
   330	            return global_ids[:32]
   331	        return None
   332
   333	    async def _run_all():
   334	        for speaker_name in SPEAKER_MAP.keys():
   335	            try:
   336	                global_ids = await _capture_globals_for_speaker(speaker_name)
   337	                if global_ids and len(global_ids) == 32:
   338	                    runtime.speaker_global_cache[speaker_name] = global_ids
   339	                    logger.info(f"  Cached globals for {speaker_name}")
   340	                else:
   341	                    logger.warning(f"  Failed to capture globals for {speaker_name}")
   342	            except Exception as e:
   343	                logger.warning(f"  Error caching globals for {speaker_name}: {e}")
   344
   345	    # Run async warmup in event loop
   346	    try:
   347	        loop = asyncio.get_event_loop()
   348	        if loop.is_running():
   349	            # Already in async context (unlikely at startup, but handle it)
   350	            import concurrent.futures
   351	            with concurrent.futures.ThreadPoolExecutor() as pool:
   352	                pool.submit(lambda: asyncio.run(_run_all())).result()
   353	        else:
   354	            loop.run_until_complete(_run_all())
   355	    except RuntimeError:
   356	        # No event loop exists yet
   357	        asyncio.run(_run_all())
   358
   359	    elapsed = (time.time() - t_start) * 1000
   360	    cached = len(runtime.speaker_global_cache)
   361	    total = len(SPEAKER_MAP)
   362	    logger.info(f"Speaker globals cached: {cached}/{total} speakers in {elapsed:.0f}ms")
   363
   364
   365	async def generate_speech(
   366	    text: str,
   367	    speaker: str,
   368	    temperature: float = 0.8,
   369	    top_k: int = 50,
   370	    top_p: float = 1.0,
   371	    max_tokens: int = 4096,
   372	    repetition_penalty: float = 1.05,
   373	    seed: Optional[int] = None,
   374	    output_sample_rate: str = "16khz",
   375	    request_id: Optional[str] = None,
   376	) -> Tuple[Optional[bytes], Dict[str, Any]]:
   377	    """
   378	    Generate speech audio (non-streaming).
   379
   380	    Args:
   381	        text: Text to synthesize (already normalized)
   382	        speaker: Internal speaker name (resolved)
   383	        temperature: Sampling temperature
   384	        top_k: Top-k sampling
   385	        top_p: Nucleus sampling
   386	        max_tokens: Maximum tokens to generate
   387	        repetition_penalty: Repetition penalty
   388	        seed: Random seed for reproducibility
   389	        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)
   390
   391	    Returns:
   392	        Tuple of (audio_bytes, metrics_dict)
   393	        audio_bytes: WAV audio bytes (16kHz or 48kHz, 16-bit PCM) or None if failed
   394	        metrics_dict: Dictionary with timing metrics
   395
   396	    Raises:
   397	        RuntimeError: If runtime not initialized
   398	    """
   399	    if not is_initialized():
   400	        raise RuntimeError("TTS runtime not initialized")
   401
   402	    runtime = get_runtime()
   403	    metrics = {
   404	        "request_id": request_id,
   405	        "ttfb_ms": 0,
   406	        "generation_ms": 0,
   407	        "tokens_generated": 0,
   408	        "audio_duration_seconds": 0.0,
   409	        "sr_applied": False,
   410	        "output_sample_rate": 16000,
   411	    }
   412
   413	    if request_id is None:
   414	        import uuid
   415	        request_id = f"runtime-ns-{uuid.uuid4().hex[:10]}"
   416	        metrics["request_id"] = request_id
   417
   418	    active_requests = _with_inflight_delta(1)
   419	    start_time = time.time()
   420
   421	    pipeline_timings = {}
   422
   423	    try:
   424	        logger.info(
   425	            "tts_runtime_nonstream_start",
   426	            extra={
   427	                "speaker": speaker,
   428	                "request_id": request_id,
   429	                "text_length": len(text),
   430	                "request_inflight": active_requests,
   431	                "temperature": temperature,
   432	                "top_k": top_k,
   433	                "top_p": top_p,
   434	                "max_tokens": max_tokens,
   435	            },
   436	        )
   437
   438	        gen_kwargs = dict(
   439	            speaker=speaker,
   440	            text=text,
   441	            temperature=temperature,
   442	            top_k=top_k,
   443	            top_p=top_p,
   444	            max_tokens=max_tokens,
   445	            repetition_penalty=repetition_penalty,
   446	            seed=seed,
   447	        )
   448
   449	        if _timing_enabled():
   450	            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
   451	                **gen_kwargs,
   452	                return_timing=True,
   453	                request_id=request_id,
   454	            )
   455	        else:
   456	            audio_bytes = await runtime.pipeline.generate_speech_indic(
   457	                **gen_kwargs,
   458	                request_id=request_id,
   459	            )
   460
   461	        generation_time = time.time() - start_time
   462	        metrics["generation_ms"] = int(generation_time * 1000)
   463…16778 tokens truncated…cts to /v1/tts/ws
    11	2. Client sends JSON: {"text": "...", "speaker": "...", ...}
    12	3. Server sends binary audio chunks
    13	4. Server sends JSON: {"event": "complete", "metrics": {...}}
    14	5. Connection closes
    15
    16	Control messages:
    17	- {"event": "cancel"} - Stop generation
    18	- {"event": "ping"} - Keep-alive (server responds with pong)
    19	"""
    20
    21	import asyncio
    22	import json
    23	import time
    24	import uuid
    25	from typing import Optional, Dict, Any
    26	from dataclasses import dataclass
    27	from enum import Enum
    28
    29	from veena3modal.shared.logging import (
    30	    get_logger,
    31	    set_request_context,
    32	    clear_request_context,
    33	    log_request_received,
    34	    log_first_audio_emitted,
    35	    log_request_completed,
    36	    log_request_failed,
    37	)
    38	from veena3modal.shared.metrics import (
    39	    record_request_received as metrics_request_received,
    40	    record_request_completed as metrics_request_completed,
    41	    record_request_failed as metrics_request_failed,
    42	    record_ttfb,
    43	    record_rtf,
    44	    record_audio_duration,
    45	    record_chunks_sent,
    46	)
    47
    48	logger = get_logger(__name__)
    49
    50
    51	class WSMessageType(str, Enum):
    52	    """WebSocket message types."""
    53	    # Client -> Server
    54	    REQUEST = "request"
    55	    CANCEL = "cancel"
    56	    PING = "ping"
    57
    58	    # Server -> Client
    59	    AUDIO_CHUNK = "audio_chunk"  # Binary frame
    60	    HEADER = "header"  # WAV header info
    61	    PROGRESS = "progress"  # Progress update
    62	    COMPLETE = "complete"  # Generation complete
    63	    ERROR = "error"  # Error occurred
    64	    PONG = "pong"  # Response to ping
    65
    66
    67	@dataclass
    68	class WSRequest:
    69	    """Parsed WebSocket TTS request."""
    70	    text: str
    71	    speaker: str = "male1"
    72	    temperature: float = 0.8
    73	    top_k: int = 50
    74	    top_p: float = 1.0
    75	    max_tokens: int = 4096
    76	    repetition_penalty: float = 1.05
    77	    seed: Optional[int] = None
    78	    chunking: bool = False
    79	    normalize: bool = True
    80	    format: str = "wav"
    81	    sample_rate: int = 16000
    82
    83	    @classmethod
    84	    def from_dict(cls, data: Dict[str, Any]) -> "WSRequest":
    85	        """Parse request from JSON dict."""
    86	        return cls(
    87	            text=data.get("text", ""),
    88	            speaker=data.get("speaker", "male1"),
    89	            temperature=data.get("temperature", 0.8),
    90	            top_k=data.get("top_k", 50),
    91	            top_p=data.get("top_p", 1.0),
    92	            max_tokens=data.get("max_tokens", 4096),
    93	            repetition_penalty=data.get("repetition_penalty", 1.05),
    94	            seed=data.get("seed"),
    95	            chunking=data.get("chunking", False),
    96	            normalize=data.get("normalize", True),
    97	            format=data.get("format", "wav"),
    98	            sample_rate=data.get("sample_rate", 16000),
    99	        )
   100
   101	    def validate(self) -> Optional[str]:
   102	        """Validate request, return error message if invalid."""
   103	        if not self.text or not self.text.strip():
   104	            return "Text is required and cannot be empty"
   105	        if len(self.text) > 50000:
   106	            return f"Text too long: {len(self.text)} chars (max 50000)"
   107	        if self.format != "wav":
   108	            return f"WebSocket streaming only supports WAV format (got {self.format})"
   109	        return None
   110
   111
   112	def create_error_message(code: str, message: str, request_id: str) -> str:
   113	    """Create JSON error message."""
   114	    return json.dumps({
   115	        "event": WSMessageType.ERROR.value,
   116	        "error": {
   117	            "code": code,
   118	            "message": message,
   119	            "request_id": request_id,
   120	        }
   121	    })
   122
   123
   124	def create_header_message(
   125	    request_id: str,
   126	    sample_rate: int,
   127	    model_version: str,
   128	) -> str:
   129	    """Create JSON header message (sent before audio)."""
   130	    return json.dumps({
   131	        "event": WSMessageType.HEADER.value,
   132	        "request_id": request_id,
   133	        "sample_rate": sample_rate,
   134	        "format": "wav",
   135	        "model_version": model_version,
   136	    })
   137
   138
   139	def create_progress_message(
   140	    chunks_sent: int,
   141	    bytes_sent: int,
   142	    elapsed_ms: int,
   143	) -> str:
   144	    """Create JSON progress message."""
   145	    return json.dumps({
   146	        "event": WSMessageType.PROGRESS.value,
   147	        "chunks_sent": chunks_sent,
   148	        "bytes_sent": bytes_sent,
   149	        "elapsed_ms": elapsed_ms,
   150	    })
   151
   152
   153	def create_complete_message(
   154	    request_id: str,
   155	    metrics: Dict[str, Any],
   156	) -> str:
   157	    """Create JSON completion message."""
   158	    return json.dumps({
   159	        "event": WSMessageType.COMPLETE.value,
   160	        "request_id": request_id,
   161	        "metrics": metrics,
   162	    })
   163
   164
   165	async def handle_websocket_tts(websocket, api_key: Optional[str] = None):
   166	    """
   167	    Handle WebSocket TTS streaming connection.
   168
   169	    This is the main handler called from FastAPI WebSocket route.
   170
   171	    Args:
   172	        websocket: FastAPI WebSocket instance
   173	        api_key: Optional API key for authentication
   174	    """
   175	    from veena3modal.api.auth import get_api_validator, hash_api_key
   176	    from veena3modal.api.rate_limiter import get_rate_limiter
   177	    from veena3modal.api.error_handlers import FeatureFlags
   178	    from veena3modal.api.schemas import resolve_speaker_name
   179	    from veena3modal.services import tts_runtime
   180
   181	    request_id = str(uuid.uuid4())
   182	    set_request_context(request_id=request_id)
   183
   184	    # Accept the WebSocket connection
   185	    await websocket.accept()
   186	    logger.info(f"WebSocket connection accepted: {request_id}")
   187
   188	    # Track cancellation
   189	    cancelled = False
   190
   191	    try:
   192	        # === AUTH CHECK (if enabled) ===
   193	        if FeatureFlags.is_auth_enabled() and api_key:
   194	            validator = get_api_validator()
   195	            key_hash = hash_api_key(api_key)
   196	            auth_result = validator.validate(key_hash)
   197
   198	            if not auth_result.is_valid:
   199	                await websocket.send_text(create_error_message(
   200	                    code=auth_result.error_code or "AUTH_FAILED",
   201	                    message=auth_result.error_message or "Authentication failed",
   202	                    request_id=request_id,
   203	                ))
   204	                await websocket.close(code=1008)  # Policy violation
   205	                return
   206
   207	            # Rate limit check
   208	            if FeatureFlags.is_rate_limiting_enabled():
   209	                rate_limiter = get_rate_limiter()
   210	                allowed, remaining, reset_after = rate_limiter.check(key_hash)
   211	                if not allowed:
   212	                    await websocket.send_text(create_error_message(
   213	                        code="RATE_LIMIT_EXCEEDED",
   214	                        message=f"Rate limit exceeded. Retry after {int(reset_after) + 1} seconds.",
   215	                        request_id=request_id,
   216	                    ))
   217	                    await websocket.close(code=1008)
   218	                    return
   219
   220	        # === WAIT FOR REQUEST ===
   221	        try:
   222	            # Wait for client to send request (with timeout)
   223	            raw_message = await asyncio.wait_for(
   224	                websocket.receive_text(),
   225	                timeout=30.0
   226	            )
   227	            message = json.loads(raw_message)
   228	        except asyncio.TimeoutError:
   229	            await websocket.send_text(create_error_message(
   230	                code="TIMEOUT",
   231	                message="No request received within 30 seconds",
   232	                request_id=request_id,
   233	            ))
   234	            await websocket.close(code=1000)
   235	            return
   236	        except json.JSONDecodeError as e:
   237	            await websocket.send_text(create_error_message(
   238	                code="INVALID_JSON",
   239	                message=f"Invalid JSON: {e}",
   240	                request_id=request_id,
   241	            ))
   242	            await websocket.close(code=1003)  # Unsupported data
   243	            return
   244
   245	        # Handle control messages
   246	        event_type = message.get("event", "request")
   247	        if event_type == "ping":
   248	            await websocket.send_text(json.dumps({"event": "pong"}))
   249	            # Wait for actual request
   250	            raw_message = await websocket.receive_text()
   251	            message = json.loads(raw_message)
   252
   253	        # === PARSE REQUEST ===
   254	        req = WSRequest.from_dict(message)
   255	        validation_error = req.validate()
   256	        if validation_error:
   257	            await websocket.send_text(create_error_message(
   258	                code="VALIDATION_ERROR",
   259	                message=validation_error,
   260	                request_id=request_id,
   260	                request_id=request_id,
   261	            ))
   262	            await websocket.close(code=1003)
   263	            return
   264
   265	        # Resolve speaker
   266	        speaker = resolve_speaker_name(req.speaker)
   267
   268	        # Log request
   269	        log_request_received(
   270	            request_id=request_id,
   271	            text_length=len(req.text),
   272	            speaker=speaker,
   273	            stream=True,
   274	            format="wav",
   275	        )
   276	        metrics_request_received(speaker=speaker, stream=True, format="wav")
   277
   278	        # === CHECK MODEL ===
   279	        if not tts_runtime.is_initialized():
   280	            await websocket.send_text(create_error_message(
   281	                code="MODEL_NOT_LOADED",
   282	                message="TTS model not initialized",
   283	                request_id=request_id,
   284	            ))
   285	            await websocket.close(code=1011)  # Internal error
   286	            return
   287
   288	        runtime = tts_runtime.get_runtime()
   289	        model_version = runtime.model_version if runtime else "unknown"
   290
   291	        # === SEND HEADER INFO ===
   292	        await websocket.send_text(create_header_message(
   293	            request_id=request_id,
   294	            sample_rate=req.sample_rate,
   295	            model_version=model_version,
   296	        ))
   297
   298	        # === STREAM AUDIO ===
   299	        start_time = time.time()
   300	        chunks_sent = 0
   301	        total_bytes = 0
   302	        first_chunk_time = None
   303	        final_metrics = {}
   304
   305	        # Start a task to listen for cancel messages
   306	        async def listen_for_cancel():
   307	            nonlocal cancelled
   308	            try:
   309	                while not cancelled:
   310	                    try:
   311	                        msg = await asyncio.wait_for(
   312	                            websocket.receive_text(),
   313	                            timeout=0.1
   314	                        )
   315	                        data = json.loads(msg)
   316	                        if data.get("event") == "cancel":
   317	                            cancelled = True
   318	                            logger.info(f"Client requested cancel: {request_id}")
   319	                            break
   320	                        elif data.get("event") == "ping":
   321	                            await websocket.send_text(json.dumps({"event": "pong"}))
   322	                    except asyncio.TimeoutError:
   323	                        continue
   324	                    except Exception:
   325	                        break
   326	            except Exception:
   327	                pass
   328
   329	        cancel_task = asyncio.create_task(listen_for_cancel())
   330
   331	        try:
   332	            # Apply text normalization if enabled
   333	            text = req.text
   334	            if req.normalize:
   335	                try:
   336	                    from veena3modal.processing.text_normalizer import normalize_text
   337	                    text = normalize_text(text)
   338	                except ImportError:
   339	                    pass  # Normalizer not available
   340
   341	            # Stream audio chunks
   342	            async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
   343	                text=text,
   344	                speaker=speaker,
   345	                temperature=req.temperature,
   346	                top_k=req.top_k,
   347	                top_p=req.top_p,
   348	                max_tokens=req.max_tokens,
   349	                repetition_penalty=req.repetition_penalty,
   350	                seed=req.seed,
   351	                enable_chunking=req.chunking,
   352	            ):
   353	                if cancelled:
   354	                    logger.info(f"Generation cancelled: {request_id}")
   355	                    break
   356
   357	                final_metrics = metrics
   358	                chunks_sent += 1
   359	                total_bytes += len(audio_chunk)
   360
   361	                # Record TTFB on first chunk
   362	                if first_chunk_time is None:
   363	                    first_chunk_time = time.time()
   364	                    ttfb_ms = int((first_chunk_time - start_time) * 1000)
   365	                    log_first_audio_emitted(
   366	                        request_id=request_id,
   367	                        ttfb_ms=ttfb_ms,
   368	                        chunk_size_bytes=len(audio_chunk),
   369	                    )
   370	                    record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)
   371
   372	                # Send audio as binary frame
   373	                await websocket.send_bytes(audio_chunk)
   374
   375	                # Send progress every 10 chunks
   376	                if chunks_sent % 10 == 0:
   377	                    await websocket.send_text(create_progress_message(
   378	                        chunks_sent=chunks_sent,
   379	                        bytes_sent=total_bytes,
   380	                        elapsed_ms=int((time.time() - start_time) * 1000),
   381	                    ))
   382
   383	        finally:
   384	            cancel_task.cancel()
   385	            try:
   386	                await cancel_task
   387	            except asyncio.CancelledError:
   388	                pass
   389
   390	        # === SEND COMPLETION ===
   391	        total_time = time.time() - start_time
   392	        audio_duration = final_metrics.get("audio_duration_seconds", 0)
   393	        rtf = total_time / audio_duration if audio_duration > 0 else 0
   394
   395	        completion_metrics = {
   396	            "request_id": request_id,
   397	            "chunks_sent": chunks_sent,
   398	            "total_bytes": total_bytes,
   399	            "audio_duration_seconds": round(audio_duration, 2),
   400	            "total_time_seconds": round(total_time, 2),
   401	            "ttfb_ms": int((first_chunk_time - start_time) * 1000) if first_chunk_time else 0,
   402	            "rtf": round(rtf, 3),
   403	            "cancelled": cancelled,
   404	        }
   405
   406	        await websocket.send_text(create_complete_message(
   407	            request_id=request_id,
   408	            metrics=completion_metrics,
   409	        ))
   410
   411	        # Log completion
   412	        if not cancelled:
   413	            log_request_completed(
   414	                request_id=request_id,
   415	                status_code=200,
   416	                total_duration_ms=int(total_time * 1000),
   417	                audio_duration_seconds=audio_duration,
   418	                rtf=rtf,
   419	                chunks_sent=chunks_sent,
   420	            )
   421	            metrics_request_completed(
   422	                status_code=200,
   423	                duration_seconds=total_time,
   424	                speaker=speaker,
   425	                stream=True,
   426	            )
   427	            record_rtf(rtf=rtf, speaker=speaker)
   428	            record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
   429	            record_chunks_sent(chunks=chunks_sent, speaker=speaker)
   430
   431	        # Close connection gracefully
   432	        await websocket.close(code=1000)
   433
   434	    except Exception as e:
   435	        logger.exception(f"WebSocket error: {e}")
   436	        log_request_failed(
   437	            request_id=request_id,
   438	            status_code=500,
   439	            error_code="WEBSOCKET_ERROR",
   440	            error_message=str(e),
   441	        )
   442	        metrics_request_failed(
   443	            status_code=500,
   444	            error_code="WEBSOCKET_ERROR",
   445	        )
   446
   447	        try:
   448	            await websocket.send_text(create_error_message(
   449	                code="INTERNAL_ERROR",
   450	                message=str(e),
   451	                request_id=request_id,
   452	            ))
   453	            await websocket.close(code=1011)
   454	        except Exception:
   455	            pass
   456
   457	    finally:
   458	        clear_request_context()
   459
   460
   461	def add_websocket_routes(app):
   462	    """
   463	    Add WebSocket routes to FastAPI app.
   464
   465	    Call this from create_app() to enable WebSocket support.
   466
   467	    Args:
   468	        app: FastAPI application instance
   469	    """
   470	    from fastapi import WebSocket, Query
   471
   472	    @app.websocket("/v1/tts/ws")
   473	    async def websocket_tts(
   474	        websocket: WebSocket,
   475	        api_key: Optional[str] = Query(None, alias="api_key"),
   476	    ):
   477	        """
   478	        WebSocket endpoint for TTS streaming.
   479
   480	        Connect with optional api_key query parameter:
   481	        ws://host/v1/tts/ws?api_key=your-key
   482
   483	        Then send JSON request:
   484	        {"text": "Hello world", "speaker": "male1"}
   485
   486	        Receive:
   487	        - JSON header message
   488	        - Binary audio chunks
   489	        - JSON progress messages (every 10 chunks)
   490	        - JSON completion message
   491	        """
   492	        # Also check Authorization header
   493	        auth_header = websocket.headers.get("authorization", "")
   494	        if auth_header.startswith("Bearer "):
   495	            api_key = auth_header[7:]
   496
   497	        await handle_websocket_tts(websocket, api_key)
   498
   499	    logger.info("WebSocket route /v1/tts/ws registered")
   500


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


cd /home/ubuntu/veenaModal && nl -ba veena3modal/services/tts_runtime.py | sed -n '420,760p'


cd /home/ubuntu/veenaModal && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '720,1040p'


cd /home/ubuntu/veenaModal && nl -ba veena3modal/api/fastapi_app.py | sed -n '520,760p'


Output:
Chunk ID: 2a8321
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 3768
Output:
   420
   421	    pipeline_timings = {}
   422
   423	    try:
   424	        logger.info(
   425	            "tts_runtime_nonstream_start",
   426	            extra={
   427	                "speaker": speaker,
   428	                "request_id": request_id,
   429	                "text_length": len(text),
   430	                "request_inflight": active_requests,
   431	                "temperature": temperature,
   432	                "top_k": top_k,
   433	                "top_p": top_p,
   434	                "max_tokens": max_tokens,
   435	            },
   436	        )
   437
   438	        gen_kwargs = dict(
   439	            speaker=speaker,
   440	            text=text,
   441	            temperature=temperature,
   442	            top_k=top_k,
   443	            top_p=top_p,
   444	            max_tokens=max_tokens,
   445	            repetition_penalty=repetition_penalty,
   446	            seed=seed,
   447	        )
   448
   449	        if _timing_enabled():
   450	            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
   451	                **gen_kwargs,
   452	                return_timing=True,
   453	                request_id=request_id,
   454	            )
   455	        else:
   456	            audio_bytes = await runtime.pipeline.generate_speech_indic(
   457	                **gen_kwargs,
   458	                request_id=request_id,
   459	            )
   460
   461	        generation_time = time.time() - start_time
   462	        metrics["generation_ms"] = int(generation_time * 1000)
   463	        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time
   464
   465	        if pipeline_timings:
   466	            metrics.update({
   467	                "llm_token_total": pipeline_timings.get("token_total", 0),
   468	                "llm_batch_count": pipeline_timings.get("batch_count", 0),
   469	                "llm_batch_gpu_ms_total": pipeline_timings.get("llm_gpu_batch_ms_total", 0),
   470	                "llm_batch_gpu_ms_min": pipeline_timings.get("llm_gpu_batch_ms_min", 0),
   471	                "llm_batch_gpu_ms_max": pipeline_timings.get("llm_gpu_batch_ms_max", 0),
   472	                "llm_batch_gpu_ms_p50": pipeline_timings.get("llm_gpu_batch_ms_p50", 0),
   473	                "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
   474	                "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
   475	                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
   476	                "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
   477	                "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
   478	                "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
   479	                "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
   480	            })
   481
   482	        if audio_bytes:
   483	            # Apply super-resolution if requested and available
   484	            sample_rate = 16000
   485	            logger.info(f"SR check: output_sample_rate={output_sample_rate!r}, "
   486	                       f"sr_service={runtime.sr_service is not None}, "
   487	                       f"is_loaded={runtime.sr_service.is_loaded if runtime.sr_service else False}")
   488
   489	            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
   490	                try:
   491	                    sr_start = time.time()
   492	                    logger.info(f"Applying super-resolution to {len(audio_bytes)} bytes...")
   493	                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
   494	                    sr_time = (time.time() - sr_start) * 1000
   495	                    metrics["sr_ms"] = int(sr_time)
   496	                    metrics["sr_applied"] = True
   497	                    sample_rate = 48000
   498	                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
   499	                except Exception as e:
   500	                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
   501	                    import traceback
   502	                    traceback.print_exc()
   503
   504	            metrics["output_sample_rate"] = sample_rate
   505
   506	            # Calculate audio duration
   507	                audio_duration = (len(audio_bytes) - 44) / (sample_rate * 2)  # -44 for WAV header
   508	                metrics["audio_duration_seconds"] = max(0.0, audio_duration)
   509
   510	        if _timing_enabled():
   511	            logger.info("tts_runtime_nonstream_complete", extra=metrics)
   512
   513	        return audio_bytes, metrics
   514
   515	    except Exception as e:
   516	        logger.error(f"Speech generation failed: {e}")
   517	        raise
   518	        finally:
   519	            active_requests = _with_inflight_delta(-1)
   520	            if _timing_enabled():
   521	                logger.info(
   522	                    "tts_runtime_nonstream_inflight",
   523	                    extra={"active_requests": active_requests},
   524	                )
   525
   526
   527	def _apply_super_resolution(audio_bytes: bytes, sr_service) -> bytes:
   528	    """
   529	    Apply super-resolution to audio bytes.
   530
   531	    Args:
   532	        audio_bytes: WAV audio at 16kHz
   533	        sr_service: SuperResolutionService instance
   534
   535	    Returns:
   536	        WAV audio at 48kHz
   537	    """
   538	    import numpy as np
   539	    import struct
   540	    import torch
   541
   542	    # Parse WAV header and extract PCM data
   543	    if len(audio_bytes) < 44:
   544	        raise ValueError("Invalid WAV data")
   545
   546	    # Extract PCM data (skip 44-byte WAV header)
   547	    pcm_data = np.frombuffer(audio_bytes[44:], dtype=np.int16)
   548
   549	    # Convert to float32 for SR model
   550	    audio_float = pcm_data.astype(np.float32) / 32768.0
   551
   552	    # Convert numpy to torch tensor for process_chunk
   553	    audio_tensor = torch.from_numpy(audio_float)
   554
   555	    # Apply super-resolution using process_chunk (16kHz -> 48kHz)
   556	    # process_chunk expects [batch, samples] or [samples] and returns [batch, samples]
   557	    upsampled_tensor = sr_service.process_chunk(audio_tensor)
   558
   559	    # Convert back to numpy, squeeze batch dim if present
   560	    upsampled = upsampled_tensor.squeeze().cpu().numpy()
   561
   562	    # Convert back to int16
   563	    upsampled_int16 = np.clip(upsampled * 32768.0, -32768, 32767).astype(np.int16)
   564
   565	    # Create new WAV header for 48kHz
   566	    sample_rate = 48000
   567	    num_channels = 1
   568	    bits_per_sample = 16
   569	    byte_rate = sample_rate * num_channels * bits_per_sample // 8
   570	    block_align = num_channels * bits_per_sample // 8
   571	    data_size = len(upsampled_int16) * 2
   572
   573	    wav_header = struct.pack(
   574	        '<4sI4s4sIHHIIHH4sI',
   575	        b'RIFF',
   576	        36 + data_size,
   577	        b'WAVE',
   578	        b'fmt ',
   579	        16,  # fmt chunk size
   580	        1,   # PCM format
   581	        num_channels,
   582	        sample_rate,
   583	        byte_rate,
   584	        block_align,
   585	        bits_per_sample,
   586	        b'data',
   587	        data_size,
   588	    )
   589
   590	    return wav_header + upsampled_int16.tobytes()
   591
   592
   593	async def generate_speech_chunked(
   594	    text: str,
   595	    speaker: str,
   596	    temperature: float = 0.8,
   597	    top_k: int = 50,
   598	    top_p: float = 1.0,
   599	    max_tokens: int = 4096,
   600	    repetition_penalty: float = 1.05,
   601	    seed: Optional[int] = None,
   602	    sample_rate: int = 16000,
   603	    output_sample_rate: str = "16khz",
   604	    request_id: Optional[str] = None,
   605	) -> Tuple[Optional[bytes], Dict[str, Any]]:
   606	    """
   607	    Generate speech with automatic text chunking for long inputs.
   608
   609	    Uses LongTextProcessor to split text and stitch audio.
   610
   611	    Args:
   612	        text: Text to synthesize
   613	        speaker: Internal speaker name
   614	        ... (same as generate_speech)
   615	        sample_rate: Internal generation sample rate (always 16000)
   616	        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)
   617
   618	    Returns:
   619	        Tuple of (audio_bytes, metrics_dict)
   620	    """
   621	    if not is_initialized():
   622	        raise RuntimeError("TTS runtime not initialized")
   623
   624	    runtime = get_runtime()
   625	    metrics = {
   626	        "ttfb_ms": 0,
   627	        "generation_ms": 0,
   628	        "chunks_processed": 0,
   629	        "audio_duration_seconds": 0.0,
   630	        "text_chunked": False,
   631	        "sr_applied": False,
   632	        "output_sample_rate": 16000,
   633	    }
   634
   635	    active_requests = _with_inflight_delta(1)
   636	    start_time = time.time()
   637
   638	    try:
   639	        if _timing_enabled():
   640	            logger.info(
   641	                "tts_runtime_chunking_start",
   642	                extra={
   643	                    "speaker": speaker,
   644	                    "text_length": len(text),
   645	                    "request_inflight": active_requests,
   646	                    "chunking": True,
   647	                },
   648	            )
   649	        # Import long text processor
   650	        from veena3modal.processing.long_text_processor import LongTextProcessor
   651
   652	        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
   653
   654	        # Check if chunking is needed
   655	        if long_processor.should_chunk(text):
   656	            metrics["text_chunked"] = True
   657	            audio_bytes = await long_processor.generate_with_chunking(
   658	                text=text,
   659	                speaker=speaker,
   660	                temperature=temperature,
   661	                top_k=top_k,
   662	                top_p=top_p,
   663	                max_tokens=max_tokens,
   664	                repetition_penalty=repetition_penalty,
   665	                seed=seed,
   666	                sample_rate=sample_rate,
   667	            )
   668	        else:
   669	            # Short text: direct generation
   670	            audio_bytes = await runtime.pipeline.generate_speech_indic(
   671	                speaker=speaker,
   672	                text=text,
   673	                temperature=temperature,
   674	                top_k=top_k,
   675	                top_p=top_p,
   676	                max_tokens=max_tokens,
   677	                repetition_penalty=repetition_penalty,
   678	                seed=seed,
   679	            )
   680
   681	        generation_time = time.time() - start_time
   682	        metrics["generation_ms"] = int(generation_time * 1000)
   683	        metrics["ttfb_ms"] = metrics["generation_ms"]
   684
   685	        if audio_bytes:
   686	            # Apply super-resolution if requested and available
   687	            final_sample_rate = 16000
   688	            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
   689	                try:
   690	                    sr_start = time.time()
   691	                    logger.info(f"Applying super-resolution to chunked audio ({len(audio_bytes)} bytes)...")
   692	                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
   693	                    sr_time = (time.time() - sr_start) * 1000
   694	                    metrics["sr_ms"] = int(sr_time)
   695	                    metrics["sr_applied"] = True
   696	                    final_sample_rate = 48000
   697	                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
   698	                except Exception as e:
   699	                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
   700	                    import traceback
   701	                    traceback.print_exc()
   702
   703	            metrics["output_sample_rate"] = final_sample_rate
   704	            audio_duration = (len(audio_bytes) - 44) / (final_sample_rate * 2)
   705	            metrics["audio_duration_seconds"] = max(0.0, audio_duration)
   706
   707	        if _timing_enabled():
   708	            metrics["active_requests"] = active_requests - 1
   709	            logger.info("tts_runtime_chunking_complete", extra=metrics)
   710	        return audio_bytes, metrics
   711
   712	    except Exception as e:
   713	        logger.error(f"Chunked speech generation failed: {e}")
   714	        raise
   715	    finally:
   716	        _with_inflight_delta(-1)
   717
   718
   719	from typing import AsyncGenerator
   720
   721
   722	async def generate_speech_streaming(
   723	    text: str,
   724	    speaker: str,
   725	    temperature: float = 0.8,
   726	    top_k: int = 50,
   727	    top_p: float = 1.0,
   728	    max_tokens: int = 4096,
   729	    repetition_penalty: float = 1.05,
   730	    seed: Optional[int] = None,
   731	    enable_chunking: bool = True,
   732	    request_id: Optional[str] = None,
   733	) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
   734	    """
   735	    Generate speech audio with true streaming (yields chunks as they're generated).
   736
   737	    This is the core streaming implementation for M4.
   738	    First yield includes WAV header + first PCM chunk.
   739	    Subsequent yields are raw PCM chunks.
   740
   741	    Args:
   742	        text: Text to synthesize (already normalized)
   743	        speaker: Internal speaker name (resolved)
   744	        temperature: Sampling temperature
   745	        top_k: Top-k sampling
   746	        top_p: Nucleus sampling
   747	        max_tokens: Maximum tokens to generate
   748	        repetition_penalty: Repetition penalty
   749	        seed: Random seed for reproducibility
   750	        enable_chunking: Enable text chunking for long inputs with voice consistency
   751
   752	    Yields:
   753	        Tuple of (audio_bytes, metrics_dict)
   754	        - First yield: WAV header (44 bytes) prepended to first PCM chunk
   755	        - Subsequent yields: Raw PCM chunks (int16, 16kHz)
   756	        - Final yield: metrics_dict has final timing info
   757
   758	    Raises:
   759	        RuntimeError: If runtime not initialized or streaming pipeline unavailable
   760	    """


Output:
Chunk ID: a6a659
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2291
Output:
   720	        - global_ids is request-scoped, passed explicitly (no shared state)
   721	        - Thread-safe: each request has its own state
   722	        - No global caching that could cause cross-request contamination
   723
   724	        Args:
   725	            speaker: Speaker name (must match first chunk)
   726	            text: Continuation text chunk to synthesize
   727	            global_ids: 32 global token IDs captured from first chunk
   728	            temperature: Sampling temperature
   729	            top_k: Top-k sampling
   730	            top_p: Nucleus sampling
   731	            max_tokens: Max tokens to generate
   732	            repetition_penalty: Prevent repetition
   733	            seed: Random seed (use same as first chunk for consistency)
   734
   735	        Yields:
   736	            Audio bytes (int16 PCM, 16kHz mono)
   737
   738	        Raises:
   739	            ValueError: If global_ids doesn't contain exactly 32 tokens
   740
   741	        Thread Safety:
   742	            Fully thread-safe. All state is request-scoped and passed explicitly.
   743	        """
   744	        import time
   745	        t_start = time.time()
   746
   747	        # Validate global tokens
   748	        EXPECTED_GLOBAL_COUNT = 32
   749	        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
   750	            raise ValueError(
   751	                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
   752	                f"global_ids must be captured from first chunk generation."
   753	            )
   754
   755	        # Build prompt WITH pre-filled global tokens
   756	        # This tells the model to skip global generation and go straight to semantic tokens
   757	        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)
   758
   759	        # Configure sampling
   760	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
   761	        sampling_params = SamplingParams(
   762	            temperature=temperature,
   763	            top_k=top_k,
   764	            top_p=top_p,
   765	            max_tokens=max_tokens,
   766	            min_tokens=DEFAULT_MIN_TOKENS,
   767	            repetition_penalty=repetition_penalty,
   768	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
   769	            skip_special_tokens=False,
   770	            seed=seed,
   771	        )
   772
   773	        # BiCodec token buffers
   774	        semantic_buffer = []
   775	        global_buffer_unused = []  # Not used in continuation, but needed for parser API
   776	        processed_token_count = 0  # Track how many vLLM tokens we've parsed
   777	        total_audio_chunks = 0
   778
   779	        # BiCodec streaming configuration
   780	        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
   781	        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
   782	        CROSSFADE_MS = 50
   783
   784	        # Generate unique request ID
   785	        import uuid
   786	        request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
   787
   788	        # Reuse pre-warmed token parser (singleton, created once in __init__)
   789	        token_parser = self.token_parser
   790
   791	        results_generator = self.model.engine.generate(
   792	            prompt=prompt,
   793	            sampling_params=sampling_params,
   794	            request_id=request_id,
   795	        )
   796
   797	        # Track state
   798	        last_decode_count = 0
   799	        total_samples_emitted_to_user = 0
   800	        previous_chunk_tail: Optional[bytes] = None
   801
   802	        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
   803	        async for request_output in results_generator:
   804	            generated_ids = request_output.outputs[0].token_ids
   805
   806	            # OPTIMIZATION: Incremental parsing (O(1) per new token)
   807	            # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
   808	            new_token_ids = generated_ids[processed_token_count:]
   809	            processed_token_count = len(generated_ids)
   810	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)
   811
   812	            semantic_count = len(semantic_buffer)
   813	            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   814	                if semantic_count - last_decode_count >= DECODE_INTERVAL:
   815	                    last_decode_count = semantic_count
   816
   817	                    # Use pre-captured global tokens for decoding
   818	                    if self.snac_decoder.enable_batching:
   819	                        audio_bytes = await self.snac_decoder.decode_single_async(
   820	                            semantic_ids=semantic_buffer,
   821	                            global_ids=global_ids,  # Use captured globals
   822	                            trim_warmup=False,
   823	                            use_sliding_window=False
   824	                        )
   825	                    else:
   826	                        audio_bytes = self.snac_decoder.decode_streaming(
   827	                            semantic_ids=semantic_buffer,
   828	                            global_ids=global_ids,  # Use captured globals
   829	                            use_sliding_window=False,
   830	                            trim_warmup=False
   831	                        )
   832
   833	                    if audio_bytes:
   834	                        total_samples_decoded = len(audio_bytes) // 2
   835
   836	                        if total_samples_decoded > total_samples_emitted_to_user:
   837	                            new_bytes_start = total_samples_emitted_to_user * 2
   838	                            new_audio_bytes = audio_bytes[new_bytes_start:]
   839
   840	                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
   841	                                previous_chunk_tail,
   842	                                new_audio_bytes,
   843	                                sample_rate_hz=16000,
   844	                                crossfade_ms=CROSSFADE_MS,
   845	                            )
   846
   847	                            samples_emitted_in_this_chunk = len(to_emit) // 2
   848	                            total_samples_emitted_to_user += samples_emitted_in_this_chunk
   849
   850	                            if to_emit:
   851	                                total_audio_chunks += 1
   852	                                yield to_emit
   853
   854	        # Final chunk processing
   855	        if len(semantic_buffer) > last_decode_count:
   856	            audio_bytes = self.snac_decoder.decode_streaming(
   857	                semantic_ids=semantic_buffer,
   858	                global_ids=global_ids,  # Use captured globals
   859	                use_sliding_window=False,
   860	                trim_warmup=False
   861	            )
   862	            if audio_bytes:
   863	                total_samples_decoded = len(audio_bytes) // 2
   864
   865	                if total_samples_decoded > total_samples_emitted_to_user:
   866	                    new_bytes_start = total_samples_emitted_to_user * 2
   867	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   868
   869	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   870	                        previous_chunk_tail,
   871	                        new_audio_bytes,
   872	                        sample_rate_hz=16000,
   873	                        crossfade_ms=CROSSFADE_MS,
   874	                    )
   875
   876	                    samples_emitted_in_final = len(to_emit) // 2
   877	                    total_samples_emitted_to_user += samples_emitted_in_final
   878
   879	                    if to_emit:
   880	                        total_audio_chunks += 1
   881	                        yield to_emit
   882
   883	        # Flush remaining tail
   884	        if previous_chunk_tail:
   885	            total_audio_chunks += 1
   886	            tail_samples = len(previous_chunk_tail) // 2
   887	            total_samples_emitted_to_user += tail_samples
   888	            yield previous_chunk_tail
   889
   890	        t_end = time.time()
   891	        audio_duration_s = total_samples_emitted_to_user / 16000
   892	        print(f"🎵 Continuation chunk complete: {audio_duration_s:.2f}s audio, {total_audio_chunks} chunks (using cached globals)")


Output:
Chunk ID: d1e5cd
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 1866
Output:
   520	                        "request_id": request_id,
   521	                    }
   522	                },
   523	                headers={"X-Request-ID": request_id},
   524	            )
   525
   526	        # Non-WAV formats not implemented yet (M5)
   527	        if req.format != AudioFormat.WAV.value:
   528	            return JSONResponse(
   529	                status_code=501,
   530	                content={
   531	                    "error": {
   532	                        "code": "FORMAT_NOT_IMPLEMENTED",
   533	                        "message": f"Streaming format '{req.format}' not yet implemented. Use format=wav.",
   534	                        "request_id": request_id,
   535	                    }
   536	                },
   537	                headers={"X-Request-ID": request_id},
   538	            )
   539
   540	        # Set request context for structured logging
   541	        set_request_context(request_id=request_id)
   542
   543	        # Create async generator for streaming response
   544	        async def audio_stream_generator():
   545	            """Yields audio chunks as they're generated."""
   546	            final_metrics = {}
   547	            first_chunk_sent = False
   548	            chunks_count = 0
   549	            try:
   550	                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
   551	                    text=text,
   552	                    speaker=speaker,
   553	                    temperature=req.temperature,
   554	                    top_k=req.top_k,
   555	                    top_p=req.top_p,
   556	                    max_tokens=req.max_tokens,
   557	                    repetition_penalty=req.repetition_penalty,
   558	                    seed=req.seed,
   559	                    enable_chunking=req.chunking,
   560	                ):
   561	                    final_metrics = metrics
   562	                    chunks_count += 1
   563
   564	                    # Fire-and-forget sentence storage after first chunk (don't block TTFB)
   565	                    if not first_chunk_sent:
   566	                        first_chunk_sent = True
   567	                        ttfb_ms = metrics.get("ttfb_ms", 0)
   568
   569	                        # Log first audio emitted (TTFB marker)
   570	                        log_first_audio_emitted(
   571	                            request_id=request_id,
   572	                            ttfb_ms=ttfb_ms,
   573	                            chunk_size_bytes=len(audio_chunk),
   574	                        )
   575	                        record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)
   576
   577	                        sentence_store = _get_sentence_store()
   578	                        sentence_store.store_fire_and_forget(
   579	                            request_id=request_id,
   580	                            text=text,
   581	                            speaker=speaker,
   582	                            stream=True,
   583	                            format=req.format,
   584	                            temperature=req.temperature,
   585	                            top_k=req.top_k,
   586	                            top_p=req.top_p,
   587	                            max_tokens=req.max_tokens,
   588	                            repetition_penalty=req.repetition_penalty,
   589	                            seed=req.seed,
   590	                            text_chunked=metrics.get("text_chunked", False),
   591	                            ttfb_ms=ttfb_ms,
   592	                            # audio_duration_seconds not known yet for streaming
   593	                        )
   594
   595	                    yield audio_chunk
   596
   597	                # Stream completed - log metrics
   598	                total_time = time.time() - start_time
   599	                audio_duration = final_metrics.get("audio_duration_seconds", 0)
   600	                rtf = total_time / audio_duration if audio_duration > 0 else 0
   601
   602	                log_request_completed(
   603	                    request_id=request_id,
   604	                    status_code=200,
   605	                    total_duration_ms=int(total_time * 1000),
   606	                    audio_duration_seconds=audio_duration,
   607	                    rtf=rtf,
   608	                    chunks_sent=chunks_count,
   609	                )
   610	                metrics_request_completed(
   611	                    status_code=200,
   612	                    duration_seconds=total_time,
   613	                    speaker=speaker,
   614	                    stream=True,
   615	                )
   616	                record_rtf(rtf=rtf, speaker=speaker)
   617	                record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
   618	                record_chunks_sent(chunks=chunks_count, speaker=speaker)
   619	                clear_request_context()
   620	            except Exception as e:
   621	                logger.exception(f"Streaming error: {e}")
   622	                # Log streaming failure
   623	                log_request_failed(
   624	                    request_id=request_id,
   625	                    status_code=500,
   626	                    error_code="STREAMING_ERROR",
   627	                    error_message=str(e),
   628	                )
   629	                metrics_request_failed(
   630	                    status_code=500,
   631	                    error_code="STREAMING_ERROR",
   632	                    speaker=speaker,
   633	                )
   634	                clear_request_context()
   635	                # Can't return error once streaming started
   636	                # Best we can do is stop yielding
   637	                return
   638
   639	        # Build response headers
   640	        # Note: Some headers can't be set dynamically during streaming
   641	        # TTFB and RTF will be logged but not in headers for streaming
   642	        model_version = runtime.model_version if runtime else get_model_version()
   643
   644	        headers = {
   645	            "X-Request-ID": request_id,
   646	            "X-Model-Version": model_version,
   647	            "X-Format": "wav",
   648	            "X-Sample-Rate": str(req.sample_rate),
   649	            "X-Stream": "true",
   650	            "X-Chunking-Enabled": str(req.chunking).lower(),
   651	            # Transfer-Encoding: chunked is set automatically by StreamingResponse
   652	        }
   653
   654	        if req.seed is not None:
   655	            headers["X-Seed"] = str(req.seed)
   656
   657	        return StreamingResponse(
   658	            audio_stream_generator(),
   659	            media_type="audio/wav",
   660	            headers=headers,
   661	        )
   662
   663	    # === WebSocket Support (M5b) ===
   664	    try:
   665	        from veena3modal.api.websocket_handler import add_websocket_routes
   666	        add_websocket_routes(app)
   667	        logger.info("WebSocket support enabled at /v1/tts/ws")
   668	    except ImportError as e:
   669	        logger.warning(f"WebSocket support not available: {e}")
   670
   671	    return app
   672
   673


Summary: auto


cd /home/ubuntu/veenaModal && nl -ba veena3modal/api/fastapi_app.py | sed -n '430,540p'


Output:
Chunk ID: ecb8a9
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 1389
Output:
   430	                        OpusEncoder, MP3Encoder, MuLawEncoder, FLACEncoder
   431	                    )
   432
   433	                    # Get sample rate from output (after potential SR)
   434	                    sample_rate = metrics.get("output_sample_rate", 16000)
   435
   436	                    # Strip WAV header to get PCM data
   437	                    pcm_data = audio_bytes[44:] if len(audio_bytes) > 44 else audio_bytes
   438
   439	                    if target_format == AudioFormat.OPUS.value:
   440	                        encoder = OpusEncoder(sample_rate=sample_rate)
   441	                        audio_bytes = encoder.encode(pcm_data, sample_rate)
   442	                        media_type = "audio/opus"
   443	                    elif target_format == AudioFormat.MP3.value:
   444	                        encoder = MP3Encoder(sample_rate=sample_rate)
   445	                        audio_bytes = encoder.encode(pcm_data, sample_rate)
   446	                        media_type = "audio/mpeg"
   447	                    elif target_format == AudioFormat.MULAW.value:
   448	                        encoder = MuLawEncoder()
   449	                        audio_bytes = encoder.encode(pcm_data, sample_rate)
   450	                        media_type = "audio/x-wav"
   451	                    elif target_format == AudioFormat.FLAC.value:
   452	                        encoder = FLACEncoder(sample_rate=sample_rate)
   453	                        audio_bytes = encoder.encode(pcm_data, sample_rate)
   454	                        media_type = "audio/flac"
   455
   456	                    headers["X-Audio-Bytes"] = str(len(audio_bytes))
   457	                    logger.info(f"Encoded audio to {target_format}: {len(audio_bytes)} bytes")
   458	                except ImportError as e:
   459	                    logger.warning(f"Audio encoder not available: {e}")
   460	                    # Fall back to WAV
   461	                except Exception as e:
   462	                    logger.error(f"Format encoding failed: {e}")
   463	                    # Fall back to WAV
   464
   465	            return Response(
   466	                content=audio_bytes,
   467	                media_type=media_type,
   468	                headers=headers,
   469	            )
   470
   471	        except Exception as e:
   472	            logger.exception(f"TTS generation error: {e}")
   473
   474	            # Log failure (lifecycle event + metrics)
   475	            log_request_failed(
   476	                request_id=request_id,
   477	                status_code=500,
   478	                error_code="INTERNAL_ERROR",
   479	                error_message=str(e),
   480	            )
   481	            metrics_request_failed(
   482	                status_code=500,
   483	                error_code="INTERNAL_ERROR",
   484	                speaker=speaker,
   485	            )
   486	            clear_request_context()
   487
   488	            return JSONResponse(
   489	                status_code=500,
   490	                content={
   491	                    "error": {
   492	                        "code": "INTERNAL_ERROR",
   493	                        "message": str(e),
   494	                        "request_id": request_id,
   495	                    }
   496	                },
   497	                headers={"X-Request-ID": request_id},
   498	            )
   499
   500	    async def _handle_streaming_request(req, request_id: str, text: str, speaker: str, start_time: float):
   501	        """
   502	        Handle streaming TTS request.
   503
   504	        Returns a StreamingResponse that yields WAV header + PCM chunks.
   505	        True streaming: first bytes sent ASAP before full audio is generated.
   506	        """
   507	        from fastapi.responses import StreamingResponse
   508	        from veena3modal.api.schemas import AudioFormat
   509	        from veena3modal.services import tts_runtime
   510
   511	        # Check if streaming pipeline is available
   512	        runtime = tts_runtime.get_runtime()
   513	        if runtime is None or runtime.streaming_pipeline is None:
   514	            return JSONResponse(
   515	                status_code=503,
   516	                content={
   517	                    "error": {
   518	                        "code": "STREAMING_UNAVAILABLE",
   519	                        "message": "Streaming pipeline not initialized.",
   520	                        "request_id": request_id,
   521	                    }
   522	                },
   523	                headers={"X-Request-ID": request_id},
   524	            )
   525
   526	        # Non-WAV formats not implemented yet (M5)
   527	        if req.format != AudioFormat.WAV.value:
   528	            return JSONResponse(
   529	                status_code=501,
   530	                content={
   531	                    "error": {
   532	                        "code": "FORMAT_NOT_IMPLEMENTED",
   533	                        "message": f"Streaming format '{req.format}' not yet implemented. Use format=wav.",
   534	                        "request_id": request_id,
   535	                    }
   536	                },
   537	                headers={"X-Request-ID": request_id},
   538	            )
   539
   540	        # Set request context for structured logging


cd /home/ubuntu/veenaModal && nl -ba veena3modal/services/tts_runtime.py | sed -n '760,980p'


cd /home/ubuntu/veenaModal && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '980,1180p'


Output:
Chunk ID: 3d9861
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 2548
Output:
   760	    """
   761	    if not is_initialized():
   762	        raise RuntimeError("TTS runtime not initialized")
   763
   764	    runtime = get_runtime()
   765
   766	    if runtime.streaming_pipeline is None:
   767	        raise RuntimeError("Streaming pipeline not available")
   768
   769	    # Import audio utils for WAV header
   770	    from veena3modal.audio.utils import create_wav_header
   771
   772	    # Metrics tracking
   773	    metrics = {
   774	        "ttfb_ms": 0,
   775	        "chunks_sent": 0,
   776	        "total_bytes": 0,
   777	        "audio_duration_seconds": 0.0,
   778	        "text_chunked": False,
   779	    }
   780
   781	    start_time = time.time()
   782	    first_chunk_time = None
   783	    total_pcm_bytes = 0
   784	    sample_rate = 16000  # BiCodec sample rate
   785	    active_requests = _with_inflight_delta(1)
   786	    if _timing_enabled():
   787	        logger.info(
   788	            "tts_runtime_stream_start",
   789	            extra={
   790	                "speaker": speaker,
   791	                "text_length": len(text),
   792	                "request_inflight": active_requests,
   793	                "chunking": enable_chunking,
   794	                "request_id": request_id,
   795	            },
   796	        )
   797
   798	    try:
   799	        # Check if we need text chunking
   800	        from veena3modal.processing.long_text_processor import LongTextProcessor
   801	        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
   802	        needs_chunking = enable_chunking and long_processor.should_chunk(text)
   803
   804	        if needs_chunking:
   805	            # Chunked streaming with voice consistency (global token caching)
   806	            metrics["text_chunked"] = True
   807	            async for audio_chunk, chunk_metrics in _stream_chunked_text(
   808	                runtime,
   809	                long_processor,
   810	                text,
   811	                speaker,
   812	                temperature,
   813	                top_k,
   814	                top_p,
   815	                max_tokens,
   816	                repetition_penalty,
   817	                seed,
   818	                sample_rate,
   819	                request_id,
   820	            ):
   821	                if first_chunk_time is None:
   822	                    first_chunk_time = time.time()
   823	                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
   824	                    # Create streaming WAV header (size=0 for unknown length)
   825	                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
   826	                    audio_chunk = wav_header + audio_chunk
   827
   828	                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
   829	                metrics["chunks_sent"] += 1
   830	                metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
   831	                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
   832	                if chunk_metrics:
   833	                    metrics.update(chunk_metrics)
   834
   835	                yield audio_chunk, metrics
   836	        else:
   837	            # Simple streaming (no text chunking)
   838	            # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
   839	            # use continuation mode to skip the ~110ms global token pre-roll.
   840	            # The model jumps straight to semantic token generation.
   841	            cached_globals = runtime.speaker_global_cache.get(speaker)
   842
   843	            if cached_globals:
   844	                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
   845	                    speaker=speaker,
   846	                    text=text,
   847	                    global_ids=cached_globals,
   848	                    temperature=temperature,
   849	                    top_k=top_k,
   850	                    top_p=top_p,
   851	                    max_tokens=max_tokens,
   852	                    repetition_penalty=repetition_penalty,
   853	                    seed=seed,
   854	                    request_id=request_id,
   855	                )
   856	            else:
   857	                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
   858	                    speaker=speaker,
   859	                    text=text,
   860	                    temperature=temperature,
   861	                    top_k=top_k,
   862	                    top_p=top_p,
   863	                    max_tokens=max_tokens,
   864	                    repetition_penalty=repetition_penalty,
   865	                    seed=seed,
   866	                    request_id=request_id,
   867	                )
   868
   869	            async for audio_chunk in stream_gen:
   870	                if first_chunk_time is None:
   871	                    first_chunk_time = time.time()
   872	                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
   873	                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
   874	                    audio_chunk = wav_header + audio_chunk
   875
   876	                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
   877	                metrics["chunks_sent"] += 1
   878	                metrics["total_bytes"] = total_pcm_bytes + 44
   879	                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
   880	                yield audio_chunk, metrics
   881	    finally:
   882	        active_requests = _with_inflight_delta(-1)
   883	        if _timing_enabled():
   884	            logger.info(
   885	                "tts_runtime_stream_end",
   886	                extra={
   887	                    "speaker": speaker,
   888	                    "request_id": request_id,
   889	                    "request_inflight": active_requests,
   890	                    "chunks_sent": metrics.get("chunks_sent", 0),
   891	                },
   892	            )
   893
   894
   895	async def _stream_chunked_text(
   896	    runtime: TTSRuntime,
   897	    long_processor,
   898	    text: str,
   899	    speaker: str,
   900	    temperature: float,
   901	    top_k: int,
   902	    top_p: float,
   903	    max_tokens: int,
   904	    repetition_penalty: float,
   905	    seed: Optional[int],
   906	    sample_rate: int,
   907	    request_id: Optional[str] = None,
   908	) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
   909	    """
   910	    Internal helper: stream chunked text with voice consistency.
   911
   912	    Uses global token caching from first chunk to maintain voice across chunks.
   913	    """
   914	    from veena3modal.audio.crossfade import crossfade_bytes_int16
   915
   916	    # Chunk text using the long text processor's chunker
   917	    chunks = long_processor.chunk_text(text)
   918
   919	    if not chunks:
   920	        return
   921
   922	    captured_globals: Optional[List[int]] = None
   923	    previous_chunk_tail: Optional[bytes] = None
   924	    chunk_metrics = {"chunks_processed": 0}
   925
   926	    for i, chunk_text in enumerate(chunks):
   927	        chunk_metrics["chunks_processed"] = i + 1
   928
   929	        if i == 0:
   930	            # First chunk: capture global tokens for voice consistency
   931	            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
   932	                speaker=speaker,
   933	                text=chunk_text,
   934	                temperature=temperature,
   935	                top_k=top_k,
   936	                top_p=top_p,
   937	                max_tokens=max_tokens,
   938	                repetition_penalty=repetition_penalty,
   939	                seed=seed,
   940	                request_id=request_id,
   941	            ):
   942	                if captured_globals is None and global_ids:
   943	                    captured_globals = global_ids
   944
   945	                # Crossfade with previous tail (inter-chunk stitching)
   946	                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   947	                    previous_chunk_tail,
   948	                    audio_bytes,
   949	                    sample_rate_hz=sample_rate,
   950	                    crossfade_ms=50,
   951	                )
   952
   953	                if to_emit:
   954	                    yield to_emit, chunk_metrics
   955	        else:
   956	            # Continuation chunks: use captured globals for voice consistency
   957	            if captured_globals is None:
   958	                logger.warning(f"No captured globals for chunk {i+1}, using regular streaming")
   959	                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
   960	                    speaker=speaker,
   961	                    text=chunk_text,
   962	                    temperature=temperature,
   963	                    top_k=top_k,
   964	                    top_p=top_p,
   965	                    max_tokens=max_tokens,
   966	                    repetition_penalty=repetition_penalty,
   967	                    seed=seed,
   968	                    request_id=request_id,
   969	                ):
   970	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   971	                        previous_chunk_tail,
   972	                        audio_bytes,
   973	                        sample_rate_hz=sample_rate,
   974	                        crossfade_ms=50,
   975	                    )
   976	                    if to_emit:
   977	                        yield to_emit, chunk_metrics
   978	            else:
   979	                # Use continuation method with cached globals
   980	                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(


Output:
Chunk ID: 163ee1
Wall time: 0.0501 seconds
Process exited with code 0
Original token count: 0
Output:


I’ve got the critical snippets; next I’m checking line lengths and signatures in each target file so I can safely apply one patch without guessing, then I’ll add the benchmark script and wire request IDs through all execution paths.


cd /home/ubuntu/veenaModal && wc -l veena3modal/core/streaming_pipeline.py veena3modal/services/tts_runtime.py veena3modal/api/fastapi_app.py veena3modal/api/websocket_handler.py && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '980,1180p'


Output:
Chunk ID: 702d52
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 46
Output:
   892 veena3modal/core/streaming_pipeline.py
  1003 veena3modal/services/tts_runtime.py
   673 veena3modal/api/fastapi_app.py
   500 veena3modal/api/websocket_handler.py
  3068 total


cd /home/ubuntu/veenaModal && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '840,892p'


Output:
Chunk ID: 9b2925
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 711
Output:
   840	                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
   841	                                previous_chunk_tail,
   842	                                new_audio_bytes,
   843	                                sample_rate_hz=16000,
   844	                                crossfade_ms=CROSSFADE_MS,
   845	                            )
   846
   847	                            samples_emitted_in_this_chunk = len(to_emit) // 2
   848	                            total_samples_emitted_to_user += samples_emitted_in_this_chunk
   849
   850	                            if to_emit:
   851	                                total_audio_chunks += 1
   852	                                yield to_emit
   853
   854	        # Final chunk processing
   855	        if len(semantic_buffer) > last_decode_count:
   856	            audio_bytes = self.snac_decoder.decode_streaming(
   857	                semantic_ids=semantic_buffer,
   858	                global_ids=global_ids,  # Use captured globals
   859	                use_sliding_window=False,
   860	                trim_warmup=False
   861	            )
   862	            if audio_bytes:
   863	                total_samples_decoded = len(audio_bytes) // 2
   864
   865	                if total_samples_decoded > total_samples_emitted_to_user:
   866	                    new_bytes_start = total_samples_emitted_to_user * 2
   867	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   868
   869	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   870	                        previous_chunk_tail,
   871	                        new_audio_bytes,
   872	                        sample_rate_hz=16000,
   873	                        crossfade_ms=CROSSFADE_MS,
   874	                    )
   875
   876	                    samples_emitted_in_final = len(to_emit) // 2
   877	                    total_samples_emitted_to_user += samples_emitted_in_final
   878
   879	                    if to_emit:
   880	                        total_audio_chunks += 1
   881	                        yield to_emit
   882
   883	        # Flush remaining tail
   884	        if previous_chunk_tail:
   885	            total_audio_chunks += 1
   886	            tail_samples = len(previous_chunk_tail) // 2
   887	            total_samples_emitted_to_user += tail_samples
   888	            yield previous_chunk_tail
   889
   890	        t_end = time.time()
   891	        audio_duration_s = total_samples_emitted_to_user / 16000
   892	        print(f"🎵 Continuation chunk complete: {audio_duration_s:.2f}s audio, {total_audio_chunks} chunks (using cached globals)")


cd /home/ubuntu/veenaModal && nl -ba veena3modal/api/websocket_handler.py | sed -n '248,268p'


Output:
Chunk ID: c4cb19
Wall time: 0.0534 seconds
Process exited with code 0
Original token count: 223
Output:
   248	            await websocket.send_text(json.dumps({"event": "pong"}))
   249	            # Wait for actual request
   250	            raw_message = await websocket.receive_text()
   251	            message = json.loads(raw_message)
   252
   253	        # === PARSE REQUEST ===
   254	        req = WSRequest.from_dict(message)
   255	        validation_error = req.validate()
   256	        if validation_error:
   257	            await websocket.send_text(create_error_message(
   258	                code="VALIDATION_ERROR",
   259	                message=validation_error,
   260	                request_id=request_id,
   261	            ))
   262	            await websocket.close(code=1003)
   263	            return
   264
   265	        # Resolve speaker
   266	        speaker = resolve_speaker_name(req.speaker)
   267
   268	        # Log request


cd /home/ubuntu/veenaModal && rg -n "logger|generate_speech_stream_indic_first_chunk|generate_speech_stream_indic_continuation|generate_speech_stream_indic\(|generate_speech_stream_indic\(" veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 888655
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 110
Output:
222:    async def generate_speech_stream_indic(
444:                                        logger.info(
503:    async def generate_speech_stream_indic_first_chunk(
519:        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.
523:            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
699:    async def generate_speech_stream_indic_continuation(


cd /home/ubuntu/veenaModal && rg -n "request_id" veena3modal/services/tts_runtime.py veena3modal/core/streaming_pipeline.py veena3modal/api/fastapi_app.py veena3modal/api/websocket_handler.py veena3modal/core/pipeline.py