Output:
Chunk ID: c8fb5d
Wall time: 0.0506 seconds
Process exited with code 0
Original token count: 2630
Output:
"""
WebSocket handler for TTS streaming.

Provides bidirectional WebSocket communication for TTS:
- Client sends JSON request with text and parameters
- Server streams audio chunks as binary frames
- Client can send control messages (cancel, ping)

Protocol:
1. Client connects to /v1/tts/ws
2. Client sends JSON: {"text": "...", "speaker": "...", ...}
3. Server sends binary audio chunks
4. Server sends JSON: {"event": "complete", "metrics": {...}}
5. Connection closes

Control messages:
- {"event": "cancel"} - Stop generation
- {"event": "ping"} - Keep-alive (server responds with pong)
"""

import asyncio
import json
import time
import uuid
from typing import Optional, Dict, Any
from dataclasses import dataclass
from enum import Enum

from veena3modal.shared.logging import (
    get_logger,
    set_request_context,
    clear_request_context,
    log_request_received,
    log_first_audio_emitted,
    log_request_completed,
    log_request_failed,
)
from veena3modal.shared.metrics import (
    record_request_received as metrics_request_received,
    record_request_completed as metrics_request_completed,
    record_request_failed as metrics_request_failed,
    record_ttfb,
    record_rtf,
    record_audio_duration,
    record_chunks_sent,
)

logger = get_logger(__name__)

class WSMessageType(str, Enum):
    """WebSocket message types."""
    # Client -> Server
    REQUEST = "request"
    CANCEL = "cancel"
    PING = "ping"

    # Server -> Client
    AUDIO_CHUNK = "audio_chunk"  # Binary frame
    HEADER = "header"  # WAV header info
    PROGRESS = "progress"  # Progress update
    COMPLETE = "complete"  # Generation complete
    ERROR = "error"  # Error occurred
    PONG = "pong"  # Response to ping

@dataclass
class WSRequest:
    """Parsed WebSocket TTS request."""
    text: str
    speaker: str = "male1"
    temperature: float = 0.8
    top_k: int = 50
    top_p: float = 1.0
    max_tokens: int = 4096
    repetition_penalty: float = 1.05
    seed: Optional[int] = None
    chunking: bool = False
    normalize: bool = True
    format: str = "wav"
    sample_rate: int = 16000

    @classmethod
    def from_dict(cls, data: Dict[str, Any]) -> "WSRequest":
        """Parse request from JSON dict."""
        return cls(
            text=data.get("text", ""),
            speaker=data.get("speaker", "male1"),
            temperature=data.get("temperature", 0.8),
            top_k=data.get("top_k", 50),
            top_p=data.get("top_p", 1.0),
            max_tokens=data.get("max_tokens", 4096),
            repetition_penalty=data.get("repetition_penalty", 1.05),
            seed=data.get("seed"),
            chunking=data.get("chunking", False),
            normalize=data.get("normalize", True),
            format=data.get("format", "wav"),
            sample_rate=data.get("sample_rate", 16000),
        )

    def validate(self) -> Optional[str]:
        """Validate request, return error message if invalid."""
        if not self.text or not self.text.strip():
            return "Text is required and cannot be empty"
        if len(self.text) > 50000:
            return f"Text too long: {len(self.text)} chars (max 50000)"
        if self.format != "wav":
            return f"WebSocket streaming only supports WAV format (got {self.format})"
        return None

def create_error_message(code: str, message: str, request_id: str) -> str:
    """Create JSON error message."""
    return json.dumps({
        "event": WSMessageType.ERROR.value,
        "error": {
            "code": code,
            "message": message,
            "request_id": request_id,
        }
    })

def create_header_message(
    request_id: str,
    sample_rate: int,
    model_version: str,
) -> str:
    """Create JSON header message (sent before audio)."""
    return json.dumps({
        "event": WSMessageType.HEADER.value,
        "request_id": request_id,
        "sample_rate": sample_rate,
        "format": "wav",
        "model_version": model_version,
    })

def create_progress_message(
    chunks_sent: int,
    bytes_sent: int,
    elapsed_ms: int,
) -> str:
    """Create JSON progress message."""
    return json.dumps({
        "event": WSMessageType.PROGRESS.value,
        "chunks_sent": chunks_sent,
        "bytes_sent": bytes_sent,
        "elapsed_ms": elapsed_ms,
    })

def create_complete_message(
    request_id: str,
    metrics: Dict[str, Any],
) -> str:
    """Create JSON completion message."""
    return json.dumps({
        "event": WSMessageType.COMPLETE.value,
        "request_id": request_id,
        "metrics": metrics,
    })

async def handle_websocket_tts(websocket, api_key: Optional[str] = None):
    """
    Handle WebSocket TTS streaming connection.

    This is the main handler called from FastAPI WebSocket route.

    Args:
        websocket: FastAPI WebSocket instance
        api_key: Optional API key for authentication
    """
    from veena3modal.api.auth import get_api_validator, hash_api_key
    from veena3modal.api.rate_limiter import get_rate_limiter
    from veena3modal.api.error_handlers import FeatureFlags
    from veena3modal.api.schemas import resolve_speaker_name
    from veena3modal.services import tts_runtime

    request_id = str(uuid.uuid4())
    set_request_context(request_id=request_id)

    # Accept the WebSocket connection
    await websocket.accept()
    logger.info(f"WebSocket connection accepted: {request_id}")

    # Track cancellation
    cancelled = False

    try:
        # === AUTH CHECK (if enabled) ===
        if FeatureFlags.is_auth_enabled() and api_key:
            validator = get_api_validator()
            key_hash = hash_api_key(api_key)
            auth_result = validator.validate(key_hash)

            if not auth_result.is_valid:
                await websocket.send_text(create_error_message(
                    code=auth_result.error_code or "AUTH_FAILED",
                    message=auth_result.error_message or "Authentication failed",
                    request_id=request_id,
                ))
                await websocket.close(code=1008)  # Policy violation
                return

            # Rate limit check
            if FeatureFlags.is_rate_limiting_enabled():
                rate_limiter = get_rate_limiter()
                allowed, remaining, reset_after = rate_limiter.check(key_hash)
                if not allowed:
                    await websocket.send_text(create_error_message(
                        code="RATE_LIMIT_EXCEEDED",
                        message=f"Rate limit exceeded. Retry after {int(reset_after) + 1} seconds.",
                        request_id=request_id,
                    ))
                    await websocket.close(code=1008)
                    return

        # === WAIT FOR REQUEST ===
        try:
            # Wait for client to send request (with timeout)
            raw_message = await asyncio.wait_for(
                websocket.receive_text(),
                timeout=30.0
            )
            message = json.loads(raw_message)
        except asyncio.TimeoutError:
            await websocket.send_text(create_error_message(
                code="TIMEOUT",
                message="No request received within 30 seconds",
                request_id=request_id,
            ))
            await websocket.close(code=1000)
            return
        except json.JSONDecodeError as e:
            await websocket.send_text(create_error_message(
                code="INVALID_JSON",
                message=f"Invalid JSON: {e}",
                request_id=request_id,
            ))
            await websocket.close(code=1003)  # Unsupported data
            return

        # Handle control messages
        event_type = message.get("event", "request")
        if event_type == "ping":
            await websocket.send_text(json.dumps({"event": "pong"}))
            # Wait for actual request
            raw_message = await websocket.receive_text()
            message = json.loads(raw_message)

        # === PARSE REQUEST ===
        req = WSRequest.from_dict(message)
        validation_error = req.validate()
        if validation_error:
            await websocket.send_text(create_error_message(
                code="VALIDATION_ERROR",
                message=validation_error,
                request_id=request_id,
            ))
            await websocket.close(code=1003)
            return

        # Resolve speaker
        speaker = resolve_speaker_name(req.speaker)

        # Log request
        log_request_received(
            request_id=request_id,
            text_length=len(req.text),
            speaker=speaker,
            stream=True,
            format="wav",
        )
        metrics_request_received(speaker=speaker, stream=True, format="wav")

        # === CHECK MODEL ===
        if not tts_runtime.is_initialized():
            await websocket.send_text(create_error_message(
                code="MODEL_NOT_LOADED",
                message="TTS model not initialized",
                request_id=request_id,
            ))
            await websocket.close(code=1011)  # Internal error
            return

        runtime = tts_runtime.get_runtime()
        model_version = runtime.model_version if runtime else "unknown"

        # === SEND HEADER INFO ===
        await websocket.send_text(create_header_message(
            request_id=request_id,
            sample_rate=req.sample_rate,
            model_version=model_version,
        ))

        # === STREAM AUDIO ===
        start_time = time.time()
        chunks_sent = 0
        total_bytes = 0
        first_chunk_time = None
        final_metrics = {}

        # Start a task to listen for cancel messages
        async def listen_for_cancel():
            nonlocal cancelled
            try:
                while not cancelled:
                    try:
                        msg = await asyncio.wait_for(
                            websocket.receive_text(),
                            timeout=0.1
                        )
                        data = json.loads(msg)
                        if data.get("event") == "cancel":
                            cancelled = True
                            logger.info(f"Client requested cancel: {request_id}")
                            break
                        elif data.get("event") == "ping":


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Planning admission metrics integration**


sed -n '240,560p' veena3modal/services/tts_runtime.py


sed -n '560,760p' veena3modal/services/tts_runtime.py


sed -n '930,1180p' veena3modal/services/tts_runtime.py


Output:
Chunk ID: fa1f2c
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 3106
Output:
                from veena3modal.core.super_resolution import SuperResolutionService
                logger.info(f"Initializing super-resolution from {sr_checkpoint_dir}...")
                sr_service = SuperResolutionService(checkpoint_dir=sr_checkpoint_dir)
                # Load the model explicitly
                if sr_service.load_model(device=device):
                    logger.info("✅ Super-resolution model loaded successfully")
                else:
                    logger.warning("Super-resolution model failed to load")
                    sr_service = None
            except Exception as e:
                logger.warning(f"Super-resolution not available: {e}")

        # Determine model version
        model_version = os.path.basename(model_path.rstrip('/'))
        if not model_version:
            model_version = "spark-tts"

        load_time = (time.time() - start_time) * 1000

        # Create runtime
        _runtime = TTSRuntime(
            model=model,
            pipeline=pipeline,
            streaming_pipeline=streaming_pipeline,
            long_text_processor=long_text_processor,
            prompt_builder=prompt_builder,
            bicodec_decoder=bicodec_decoder,
            sr_service=sr_service,
            model_version=model_version,
            is_loaded=True,
            load_time_ms=load_time,
            model_path=model_path,
            bicodec_path=bicodec_path,
            sr_checkpoint_dir=sr_checkpoint_dir,
            device=device,
        )

        logger.info(f"✅ TTS runtime initialized in {load_time:.0f}ms")
        logger.info(f"   Model version: {model_version}")

        # Pre-compute speaker globals only when explicitly enabled.
        # Startup warmup runs async generation and can bind AsyncLLMEngine state
        # to a non-serving event loop if executed in a different loop context.
        if precompute_speaker_globals and num_engines <= 1:
            logger.info("Speaker globals startup precompute enabled")
            _precompute_speaker_globals(_runtime)
        elif num_engines > 1:
            logger.info("Speaker globals cache startup precompute skipped (multi-engine mode)")
        else:
            logger.info("Speaker globals cache startup precompute skipped (disabled)")

        # Update FastAPI app with model version
        try:
            from veena3modal.api.fastapi_app import set_model_version
            set_model_version(model_version)
        except ImportError:
            pass

        return _runtime

    except Exception as e:
        logger.error(f"❌ Failed to initialize TTS runtime: {e}")
        import traceback
        traceback.print_exc()
        raise RuntimeError(f"TTS runtime initialization failed: {e}") from e

def _precompute_speaker_globals(runtime: TTSRuntime) -> None:
    """
    Pre-compute and cache global tokens for all 12 speakers at startup.

    OPTIMIZATION: BiCodec streaming requires 32 "global tokens" before any audio can be emitted.
    These encode speaker identity via FSQ quantization. Since we have only 12 fixed speakers,
    we generate one short utterance per speaker at startup, capture the 32 global tokens,
    and cache them. During streaming, we inject cached globals via build_prefix_with_globals(),
    skipping the ~110ms global token pre-roll phase entirely.

    This runs synchronously at startup (adds ~5-10s to cold start, saves ~110ms per streaming TTFB).
    """
    import asyncio
    import re
    from vllm import SamplingParams
    from veena3modal.core.constants import (
        SPEAKER_MAP, TRAINING_STOP_TOKEN_IDS,
        DEFAULT_TEMPERATURE, DEFAULT_TOP_K, DEFAULT_TOP_P,
    )

    logger.info("Pre-computing global tokens for all speakers...")
    t_start = time.time()

    test_text = "Hello, this is a voice test."

    sampling_params = SamplingParams(
        temperature=DEFAULT_TEMPERATURE,
        top_k=DEFAULT_TOP_K,
        top_p=DEFAULT_TOP_P,
        max_tokens=128,  # Only need ~60 tokens (32 global + some semantic)
        stop=TRAINING_STOP_TOKEN_IDS,
        skip_special_tokens=False,
    )

    async def _capture_globals_for_speaker(speaker_name: str) -> Optional[List[int]]:
        """Generate a short utterance and capture the 32 global tokens."""
        prompt = runtime.prompt_builder.build_prefix(speaker_name, test_text)
        request_id = f"warmup-{speaker_name}-{int(time.time() * 1000)}"

        final_output = None
        async for request_output in runtime.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        ):
            final_output = request_output

        if final_output is None:
            return None

        generated_text = final_output.outputs[0].text
        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
        global_ids = [int(t) for t in global_matches]

        if len(global_ids) >= 32:
            return global_ids[:32]
        return None

    async def _run_all():
        for speaker_name in SPEAKER_MAP.keys():
            try:
                global_ids = await _capture_globals_for_speaker(speaker_name)
                if global_ids and len(global_ids) == 32:
                    runtime.speaker_global_cache[speaker_name] = global_ids
                    logger.info(f"  Cached globals for {speaker_name}")
                else:
                    logger.warning(f"  Failed to capture globals for {speaker_name}")
            except Exception as e:
                logger.warning(f"  Error caching globals for {speaker_name}: {e}")

    # Run async warmup in event loop
    try:
        loop = asyncio.get_event_loop()
        if loop.is_running():
            # Already in async context (unlikely at startup, but handle it)
            import concurrent.futures
            with concurrent.futures.ThreadPoolExecutor() as pool:
                pool.submit(lambda: asyncio.run(_run_all())).result()
        else:
            loop.run_until_complete(_run_all())
    except RuntimeError:
        # No event loop exists yet
        asyncio.run(_run_all())

    elapsed = (time.time() - t_start) * 1000
    cached = len(runtime.speaker_global_cache)
    total = len(SPEAKER_MAP)
    logger.info(f"Speaker globals cached: {cached}/{total} speakers in {elapsed:.0f}ms")

def _get_long_text_processor(runtime: TTSRuntime):
    """Return cached LongTextProcessor, creating it lazily if needed."""
    if runtime.long_text_processor is None:
        from veena3modal.processing.long_text_processor import LongTextProcessor

        runtime.long_text_processor = LongTextProcessor(
            pipeline=runtime.pipeline,
            streaming_pipeline=runtime.streaming_pipeline,
        )
    return runtime.long_text_processor

async def generate_speech(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    output_sample_rate: str = "16khz",
) -> Tuple[Optional[bytes], Dict[str, Any]]:
    """
    Generate speech audio (non-streaming).

    Args:
        text: Text to synthesize (already normalized)
        speaker: Internal speaker name (resolved)
        temperature: Sampling temperature
        top_k: Top-k sampling
        top_p: Nucleus sampling
        max_tokens: Maximum tokens to generate
        repetition_penalty: Repetition penalty
        seed: Random seed for reproducibility
        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)

    Returns:
        Tuple of (audio_bytes, metrics_dict)
        audio_bytes: WAV audio bytes (16kHz or 48kHz, 16-bit PCM) or None if failed
        metrics_dict: Dictionary with timing metrics

    Raises:
        RuntimeError: If runtime not initialized
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()
    metrics = {
        "ttfb_ms": 0,
        "generation_ms": 0,
        "tokens_generated": 0,
        "audio_duration_seconds": 0.0,
        "sr_applied": False,
        "output_sample_rate": 16000,
    }

    start_time = time.time()

    try:
        # Generate audio using pipeline (always at 16kHz)
        audio_bytes, perf = await runtime.pipeline.generate_speech_profiled(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
        )
        if perf:
            metrics.update(perf)

        generation_time = time.time() - start_time
        metrics["generation_ms"] = int(generation_time * 1000)
        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time

        if audio_bytes:
            # Apply super-resolution if requested and available
            sample_rate = 16000
            if output_sample_rate == "48khz" or logger.isEnabledFor(logging.DEBUG):
                logger.debug(
                    "SR check: output_sample_rate=%r, sr_service=%s, is_loaded=%s",
                    output_sample_rate,
                    runtime.sr_service is not None,
                    runtime.sr_service.is_loaded if runtime.sr_service else False,
                )

            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
                try:
                    sr_start = time.time()
                    logger.info(f"Applying super-resolution to {len(audio_bytes)} bytes...")
                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
                    sr_time = (time.time() - sr_start) * 1000
                    metrics["sr_ms"] = int(sr_time)
                    metrics["sr_applied"] = True
                    sample_rate = 48000
                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
                except Exception as e:
                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
                    import traceback
                    traceback.print_exc()

            metrics["output_sample_rate"] = sample_rate

            # Calculate audio duration
            audio_duration = (len(audio_bytes) - 44) / (sample_rate * 2)  # -44 for WAV header
            metrics["audio_duration_seconds"] = max(0.0, audio_duration)

        return audio_bytes, metrics

    except Exception as e:
        logger.error(f"Speech generation failed: {e}")
        raise

def _apply_super_resolution(audio_bytes: bytes, sr_service) -> bytes:
    """
    Apply super-resolution to audio bytes.

    Args:
        audio_bytes: WAV audio at 16kHz
        sr_service: SuperResolutionService instance

    Returns:
        WAV audio at 48kHz
    """
    import numpy as np
    import struct
    import torch

    # Parse WAV header and extract PCM data
    if len(audio_bytes) < 44:
        raise ValueError("Invalid WAV data")

    # Extract PCM data (skip 44-byte WAV header)
    pcm_data = np.frombuffer(audio_bytes[44:], dtype=np.int16)

    # Convert to float32 for SR model
    audio_float = pcm_data.astype(np.float32) / 32768.0

    # Convert numpy to torch tensor for process_chunk
    audio_tensor = torch.from_numpy(audio_float)

    # Apply super-resolution using process_chunk (16kHz -> 48kHz)
    # process_chunk expects [batch, samples] or [samples] and returns [batch, samples]
    upsampled_tensor = sr_service.process_chunk(audio_tensor)

    # Convert back to numpy, squeeze batch dim if present
    upsampled = upsampled_tensor.squeeze().cpu().numpy()

    # Convert back to int16
    upsampled_int16 = np.clip(upsampled * 32768.0, -32768, 32767).astype(np.int16)

    # Create new WAV header for 48kHz
    sample_rate = 48000
    num_channels = 1
    bits_per_sample = 16
    byte_rate = sample_rate * num_channels * bits_per_sample // 8
    block_align = num_channels * bits_per_sample // 8
    data_size = len(upsampled_int16) * 2


Output:
Chunk ID: 759557
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 1673
Output:

    wav_header = struct.pack(
        '<4sI4s4sIHHIIHH4sI',
        b'RIFF',
        36 + data_size,
        b'WAVE',
        b'fmt ',
        16,  # fmt chunk size
        1,   # PCM format
        num_channels,
        sample_rate,
        byte_rate,
        block_align,
        bits_per_sample,
        b'data',
        data_size,
    )

    return wav_header + upsampled_int16.tobytes()

async def generate_speech_chunked(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    sample_rate: int = 16000,
    output_sample_rate: str = "16khz",
) -> Tuple[Optional[bytes], Dict[str, Any]]:
    """
    Generate speech with automatic text chunking for long inputs.

    Uses LongTextProcessor to split text and stitch audio.

    Args:
        text: Text to synthesize
        speaker: Internal speaker name
        ... (same as generate_speech)
        sample_rate: Internal generation sample rate (always 16000)
        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)

    Returns:
        Tuple of (audio_bytes, metrics_dict)
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()
    metrics = {
        "ttfb_ms": 0,
        "generation_ms": 0,
        "chunks_processed": 0,
        "audio_duration_seconds": 0.0,
        "text_chunked": False,
        "sr_applied": False,
        "output_sample_rate": 16000,
    }

    start_time = time.time()

    try:
        long_processor = _get_long_text_processor(runtime)

        # Check if chunking is needed
        if long_processor.should_chunk(text):
            metrics["text_chunked"] = True
            audio_bytes = await long_processor.generate_with_chunking(
                text=text,
                speaker=speaker,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
                sample_rate=sample_rate,
            )
        else:
            # Short text: direct generation
            audio_bytes, perf = await runtime.pipeline.generate_speech_profiled(
                speaker=speaker,
                text=text,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
            )
            if perf:
                metrics.update(perf)

        generation_time = time.time() - start_time
        metrics["generation_ms"] = int(generation_time * 1000)
        metrics["ttfb_ms"] = metrics["generation_ms"]

        if audio_bytes:
            # Apply super-resolution if requested and available
            final_sample_rate = 16000
            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
                try:
                    sr_start = time.time()
                    logger.info(f"Applying super-resolution to chunked audio ({len(audio_bytes)} bytes)...")
                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
                    sr_time = (time.time() - sr_start) * 1000
                    metrics["sr_ms"] = int(sr_time)
                    metrics["sr_applied"] = True
                    final_sample_rate = 48000
                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
                except Exception as e:
                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
                    import traceback
                    traceback.print_exc()

            metrics["output_sample_rate"] = final_sample_rate
            audio_duration = (len(audio_bytes) - 44) / (final_sample_rate * 2)
            metrics["audio_duration_seconds"] = max(0.0, audio_duration)

        return audio_bytes, metrics

    except Exception as e:
        logger.error(f"Chunked speech generation failed: {e}")
        raise

from typing import AsyncGenerator

async def generate_speech_streaming(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    enable_chunking: bool = True,
) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
    """
    Generate speech audio with true streaming (yields chunks as they're generated).

    This is the core streaming implementation for M4.
    First yield includes WAV header + first PCM chunk.
    Subsequent yields are raw PCM chunks.

    Args:
        text: Text to synthesize (already normalized)
        speaker: Internal speaker name (resolved)
        temperature: Sampling temperature
        top_k: Top-k sampling
        top_p: Nucleus sampling
        max_tokens: Maximum tokens to generate
        repetition_penalty: Repetition penalty
        seed: Random seed for reproducibility
        enable_chunking: Enable text chunking for long inputs with voice consistency

    Yields:
        Tuple of (audio_bytes, metrics_dict)
        - First yield: WAV header (44 bytes) prepended to first PCM chunk
        - Subsequent yields: Raw PCM chunks (int16, 16kHz)
        - Final yield: metrics_dict has final timing info

    Raises:
        RuntimeError: If runtime not initialized or streaming pipeline unavailable
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()

    if runtime.streaming_pipeline is None:
        raise RuntimeError("Streaming pipeline not available")

    # Import audio utils for WAV header
    from veena3modal.audio.utils import create_wav_header

    # Metrics tracking
    metrics = {
        "ttfb_ms": 0,
        "chunks_sent": 0,
        "total_bytes": 0,
        "audio_duration_seconds": 0.0,
        "text_chunked": False,
    }

    start_time = time.time()
    first_chunk_time = None
    total_pcm_bytes = 0
    sample_rate = 16000  # BiCodec sample rate

    # Check if we need text chunking
    long_processor = _get_long_text_processor(runtime)
    needs_chunking = enable_chunking and long_processor.should_chunk(text)

    if needs_chunking:
        # Chunked streaming with voice consistency (global token caching)


Output:
Chunk ID: f1f9c8
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 241
Output:
                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                    speaker=speaker,
                    text=chunk_text,
                    global_ids=captured_globals,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                ):
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        audio_bytes,
                        sample_rate_hz=sample_rate,
                        crossfade_ms=50,
                    )
                    if to_emit:
                        yield to_emit, chunk_metrics

    # Flush remaining tail
    if previous_chunk_tail:
        yield previous_chunk_tail, chunk_metrics


Summary: auto


**Inspecting file end and line count**


wc -l veena3modal/services/tts_runtime.py && tail -n 80 veena3modal/services/tts_runtime.py


Output:
Chunk ID: a93994
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 851
Output:
952 veena3modal/services/tts_runtime.py

    captured_globals: Optional[List[int]] = None
    previous_chunk_tail: Optional[bytes] = None
    chunk_metrics = {"chunks_processed": 0}

    for i, chunk_text in enumerate(chunks):
        chunk_metrics["chunks_processed"] = i + 1

        if i == 0:
            # First chunk: capture global tokens for voice consistency
            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
                speaker=speaker,
                text=chunk_text,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
            ):
                if captured_globals is None and global_ids:
                    captured_globals = global_ids

                # Crossfade with previous tail (inter-chunk stitching)
                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                    previous_chunk_tail,
                    audio_bytes,
                    sample_rate_hz=sample_rate,
                    crossfade_ms=50,
                )

                if to_emit:
                    yield to_emit, chunk_metrics
        else:
            # Continuation chunks: use captured globals for voice consistency
            if captured_globals is None:
                logger.warning(f"No captured globals for chunk {i+1}, using regular streaming")
                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
                    speaker=speaker,
                    text=chunk_text,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                ):
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        audio_bytes,
                        sample_rate_hz=sample_rate,
                        crossfade_ms=50,
                    )
                    if to_emit:
                        yield to_emit, chunk_metrics
            else:
                # Use continuation method with cached globals
                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                    speaker=speaker,
                    text=chunk_text,
                    global_ids=captured_globals,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                ):
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        audio_bytes,
                        sample_rate_hz=sample_rate,
                        crossfade_ms=50,
                    )
                    if to_emit:
                        yield to_emit, chunk_metrics

    # Flush remaining tail
    if previous_chunk_tail:
        yield previous_chunk_tail, chunk_metrics


**Inspecting streaming metrics integration**


rg -n "timeline_to_first_chunk_emitted_ms|perf\[|yield .*metrics|emit_progress|decode_pending_requests|queue" veena3modal/core/streaming_pipeline.py -S


sed -n '380,760p' veena3modal/core/streaming_pipeline.py


sed -n '760,1320p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: e96d35
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 811
Output:
190:        perf[f"{prefix}_total"] = float(sum(values))
191:        perf[f"{prefix}_min"] = float(min(values))
192:        perf[f"{prefix}_max"] = float(max(values))
193:        perf[f"{prefix}_p50"] = float(self._p50(values))
360:        emit_progress: bool = False,
459:        decode_pending_requests: List[float] = []
534:                        decode_pending_requests.append(float(pending_requests))
596:                                    if emit_progress:
641:                        if emit_progress:
652:            if emit_progress:
705:        self._apply_stats(perf, "decode_pending_requests", decode_pending_requests)
707:        perf["bicodec_decode_wall_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))
708:        perf["bicodec_decode_gpu_ms"] = 0.0
709:        perf["bicodec_decode_cpu_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))
712:            perf["llm_time_per_token_ms"] = float(llm_generation_wall_ms) / float(total_generated_tokens)
714:            perf["llm_time_per_batch_wall_ms"] = float(sum(llm_batch_wall_ms)) / float(len(llm_batch_wall_ms))
716:            perf["llm_time_per_batch_gpu_ms"] = float(sum(llm_batch_gpu_ms)) / float(len(llm_batch_gpu_ms))
722:            perf[f"timeline_{stage}_ms"] = float(mark.get("t_ms", 0.0))
723:            perf[f"timeline_{stage}_delta_ms"] = float(mark.get("dt_ms", 0.0))
729:            perf["timeline_to_first_batch_ms"] = float(tl_first_batch)
731:            perf["timeline_to_first_chunk_emitted_ms"] = float(tl_first_emit)
733:            perf["timeline_first_batch_to_first_chunk_emitted_ms"] = float(tl_first_emit - tl_first_batch)
735:            perf["timeline_first_chunk_to_request_done_ms"] = float(tl_done - tl_first_emit)
738:            "time_in_queue": "llm_time_in_queue_ms",
746:                perf[dst] = float(final_request_metrics[src]) * 1000.0
755:        queued_ts = final_request_metrics.get("queued_ts")
759:        q_to_sched = _delta_ms(queued_ts, scheduled_ts)
762:        queued_to_last = _delta_ms(queued_ts, last_token_ts)
764:            perf["llm_queued_to_scheduled_ms"] = q_to_sched
765:            perf["llm_time_in_queue_ms"] = q_to_sched
767:            perf["llm_scheduled_to_first_token_ms"] = sched_to_first
768:            perf["llm_scheduler_ms"] = sched_to_first
770:            perf["llm_first_to_last_token_ms"] = first_to_last
771:            perf["llm_model_execute_ms"] = first_to_last
772:        if queued_to_last is not None:
773:            perf["llm_queued_to_last_token_ms"] = queued_to_last
774:            perf["llm_request_lifecycle_ms"] = queued_to_last
778:            perf["audio_duration_seconds"],
779:            perf["ttfb_ms"],
780:            perf["chunks_sent"],
781:            perf["llm_token_total"],
784:        if emit_progress:
1003:        emit_progress: bool = False,
1101:        decode_pending_requests: List[float] = []
1127:                    decode_pending_requests.append(float(pending_requests))
1180:                                if emit_progress:
1215:                        if emit_progress:
1227:            if emit_progress:
1240:        if emit_progress:
1263:            self._apply_stats(perf, "decode_pending_requests", decode_pending_requests)


Output:
Chunk ID: 935229
Wall time: 0.0516 seconds
Process exited with code 0
Original token count: 4812
Output:
        Yields:
            Audio bytes (int16 PCM, 16kHz mono - BiCodec)
        """
        import time
        import uuid

        t_request_start = time.perf_counter()
        timeline_marks: List[Dict[str, float]] = [{"stage": "request_start", "t_ms": 0.0, "dt_ms": 0.0}]
        last_mark_ms = 0.0
        mark_stage_set: set[str] = {"request_start"}

        def _mark(stage: str) -> float:
            nonlocal last_mark_ms
            now_ms = (time.perf_counter() - t_request_start) * 1000.0
            dt_ms = now_ms - last_mark_ms
            last_mark_ms = now_ms
            timeline_marks.append({"stage": stage, "t_ms": float(now_ms), "dt_ms": float(dt_ms)})
            mark_stage_set.add(stage)
            return float(now_ms)

        prompt_t0 = time.perf_counter()
        prompt = self.prompt_builder.build_prefix(speaker, text)
        prompt_build_ms = (time.perf_counter() - prompt_t0) * 1000.0
        _mark("prompt_ready")

        sampling_t0 = time.perf_counter()
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
        )
        sampling_params_ms = (time.perf_counter() - sampling_t0) * 1000.0
        _mark("sampling_ready")

        request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
        token_parser = self.token_parser
        if token_parser is None:
            tokenizer = getattr(self.model, "tokenizer", None) or getattr(self.model.engine, "tokenizer", None)
            token_parser = BiCodecTokenParser(tokenizer) if tokenizer else None
        if token_parser is None:
            raise RuntimeError("BiCodecTokenParser unavailable for streaming")

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        semantic_buffer: List[int] = []
        global_buffer: List[int] = []
        processed_token_count = 0
        total_generated_tokens = 0
        total_audio_chunks = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None
        last_decode_count = 0
        last_decode_end_ts: Optional[float] = None
        t_first_chunk_decode_done: Optional[float] = None
        t_first_chunk_emitted: Optional[float] = None
        final_request_metrics: Dict[str, float] = {}
        prev_gpu_cumulative_s: Optional[float] = None

        llm_batch_wall_ms: List[float] = []
        llm_batch_gpu_ms: List[float] = []
        llm_decode_wall_ms: List[float] = []
        llm_decode_gpu_ms: List[float] = []
        llm_parse_ms_samples: List[float] = []
        tokens_per_batch: List[float] = []
        bicodec_decode_wall_ms: List[float] = []
        bicodec_decode_tokens: List[float] = []
        bicodec_decode_intervals_ms: List[float] = []
        decode_interval_applied_tokens: List[float] = []
        decode_pending_requests: List[float] = []

        t_llm_start = time.perf_counter()
        t_last_batch = t_llm_start

        async for request_output in results_generator:
            now = time.perf_counter()
            batch_wall = (now - t_last_batch) * 1000.0
            t_last_batch = now

            outputs = request_output.outputs or []
            if not outputs:
                continue

            generated_ids = outputs[0].token_ids
            if _STREAM_USE_DELTA_OUTPUT:
                new_token_ids = generated_ids
                processed_token_count += len(new_token_ids)
            else:
                new_token_ids = generated_ids[processed_token_count:]
                processed_token_count = len(generated_ids)

            new_token_count = len(new_token_ids)
            if new_token_count <= 0:
                continue

            total_generated_tokens += new_token_count
            llm_batch_wall_ms.append(float(batch_wall))
            tokens_per_batch.append(float(new_token_count))

            if "llm_first_batch" not in mark_stage_set:
                _mark("llm_first_batch")
            elif llm_batch_wall_ms:
                llm_decode_wall_ms.append(float(batch_wall))

            raw_req_metrics = self._extract_request_metrics(getattr(request_output, "metrics", None))
            if raw_req_metrics:
                final_request_metrics = raw_req_metrics
                cumulative_gpu_s: Optional[float] = None
                if "model_execute_time" in raw_req_metrics:
                    cumulative_gpu_s = raw_req_metrics["model_execute_time"]
                elif "model_forward_time" in raw_req_metrics:
                    cumulative_gpu_s = raw_req_metrics["model_forward_time"]

                if cumulative_gpu_s is not None:
                    if prev_gpu_cumulative_s is None:
                        gpu_delta_ms = cumulative_gpu_s * 1000.0
                    elif cumulative_gpu_s >= prev_gpu_cumulative_s:
                        gpu_delta_ms = (cumulative_gpu_s - prev_gpu_cumulative_s) * 1000.0
                    else:
                        gpu_delta_ms = cumulative_gpu_s * 1000.0
                    prev_gpu_cumulative_s = cumulative_gpu_s
                    if gpu_delta_ms >= 0.0:
                        llm_batch_gpu_ms.append(float(gpu_delta_ms))
                        if len(llm_batch_wall_ms) > 1:
                            llm_decode_gpu_ms.append(float(gpu_delta_ms))

            parse_t0 = time.perf_counter()
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
            llm_parse_ms_samples.append((time.perf_counter() - parse_t0) * 1000.0)

            if len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
                semantic_count = len(semantic_buffer)
                if semantic_count >= _STREAM_MIN_SEMANTIC_FIRST_CHUNK:
                    if "first_chunk_ready" not in mark_stage_set:
                        _mark("first_chunk_ready")

                    decode_interval, pending_requests = self._resolve_decode_interval(
                        first_chunk_emitted=t_first_chunk_emitted is not None
                    )
                    semantic_delta = semantic_count - last_decode_count
                    if semantic_delta >= decode_interval:
                        decode_step_tokens = semantic_delta
                        last_decode_count = semantic_count
                        decode_interval_applied_tokens.append(float(decode_interval))
                        decode_pending_requests.append(float(pending_requests))
                        decode_global = global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT]

                        if (
                            _STREAM_WINDOWED_DECODE
                            and semantic_count > _STREAM_WINDOW_SIZE
                        ):
                            decode_semantic = semantic_buffer[-_STREAM_WINDOW_SIZE:]
                        else:
                            decode_semantic = semantic_buffer

                        decode_t0 = time.perf_counter()
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=decode_semantic,
                            global_ids=decode_global,
                            trim_warmup=False,
                            use_sliding_window=False,
                        )
                        decode_t1 = time.perf_counter()

                        decode_wall_ms = (decode_t1 - decode_t0) * 1000.0
                        bicodec_decode_wall_ms.append(float(decode_wall_ms))
                        bicodec_decode_tokens.append(float(len(decode_semantic)))
                        if last_decode_end_ts is not None:
                            bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
                        last_decode_end_ts = decode_t1

                        if audio_bytes:
                            total_samples_decoded = len(audio_bytes) // 2
                            if (
                                _STREAM_WINDOWED_DECODE
                                and semantic_count > _STREAM_WINDOW_SIZE
                            ):
                                new_samples_approx = decode_step_tokens * 320
                                if total_samples_decoded > new_samples_approx:
                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
                                else:
                                    new_bytes_start = 0
                                new_audio_bytes = audio_bytes[new_bytes_start:]
                            else:
                                if total_samples_decoded > total_samples_emitted_to_user:
                                    new_bytes_start = total_samples_emitted_to_user * 2
                                    new_audio_bytes = audio_bytes[new_bytes_start:]
                                else:
                                    new_audio_bytes = b""

                            if new_audio_bytes:
                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                    previous_chunk_tail,
                                    new_audio_bytes,
                                    sample_rate_hz=16000,
                                    crossfade_ms=_STREAM_CROSSFADE_MS,
                                )
                                if to_emit:
                                    total_audio_chunks += 1
                                    total_samples_emitted_to_user += len(to_emit) // 2
                                    if t_first_chunk_decode_done is None:
                                        t_first_chunk_decode_done = decode_t1
                                        _mark("first_chunk_decoded")
                                    if t_first_chunk_emitted is None:
                                        t_first_chunk_emitted = time.perf_counter()
                                        _mark("first_chunk_emitted")
                                    if emit_progress:
                                        yield (to_emit, {})
                                    else:
                                        yield to_emit

        llm_generation_wall_ms = (time.perf_counter() - t_llm_start) * 1000.0
        _mark("llm_done")

        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
            decode_semantic = semantic_buffer
            decode_t0 = time.perf_counter()
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=decode_semantic,
                global_ids=global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT],
                use_sliding_window=False,
                trim_warmup=False,
            )
            decode_t1 = time.perf_counter()
            decode_wall_ms = (decode_t1 - decode_t0) * 1000.0
            bicodec_decode_wall_ms.append(float(decode_wall_ms))
            bicodec_decode_tokens.append(float(len(decode_semantic)))
            if last_decode_end_ts is not None:
                bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
            last_decode_end_ts = decode_t1

            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2
                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=_STREAM_CROSSFADE_MS,
                    )
                    if to_emit:
                        total_audio_chunks += 1
                        total_samples_emitted_to_user += len(to_emit) // 2
                        if t_first_chunk_decode_done is None:
                            t_first_chunk_decode_done = decode_t1
                            _mark("first_chunk_decoded")
                        if t_first_chunk_emitted is None:
                            t_first_chunk_emitted = time.perf_counter()
                            _mark("first_chunk_emitted")
                        if emit_progress:
                            yield (to_emit, {})
                        else:
                            yield to_emit

        if previous_chunk_tail:
            total_audio_chunks += 1
            total_samples_emitted_to_user += len(previous_chunk_tail) // 2
            if t_first_chunk_emitted is None:
                t_first_chunk_emitted = time.perf_counter()
                _mark("first_chunk_emitted")
            if emit_progress:
                yield (previous_chunk_tail, {})
            else:
                yield previous_chunk_tail

        _mark("stream_done")
        _mark("request_done")

        perf: Dict[str, Any] = {
            "prompt_build_ms": float(prompt_build_ms),
            "sampling_params_ms": float(sampling_params_ms),
            "llm_generation_wall_ms": float(llm_generation_wall_ms),
            "llm_batch_count": int(len(llm_batch_wall_ms)),
            "llm_token_total": int(total_generated_tokens),
            "semantic_token_total": int(len(semantic_buffer)),
            "global_token_total": int(len(global_buffer)),
            "llm_parse_ms": float(sum(llm_parse_ms_samples)),
            "bicodec_decode_calls": int(len(bicodec_decode_wall_ms)),
            "llm_decode_calls": int(len(bicodec_decode_wall_ms)),
            "chunks_sent": int(total_audio_chunks),
            "audio_duration_seconds": float(total_samples_emitted_to_user / 16000.0),
            "ttfb_ms": float((t_first_chunk_emitted - t_request_start) * 1000.0 if t_first_chunk_emitted else 0.0),
            "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
            "stream_decode_interval_tokens": int(_STREAM_DECODE_INTERVAL),
            "stream_decode_interval_first_tokens": int(_STREAM_DECODE_INTERVAL_FIRST),
            "stream_decode_interval_busy_tokens": int(_STREAM_DECODE_INTERVAL_BUSY),
            "stream_decode_busy_pending_threshold": int(_STREAM_DECODE_BUSY_PENDING),
            "stream_adaptive_decode": bool(_STREAM_ADAPTIVE_DECODE),
            "stream_window_size_tokens": int(_STREAM_WINDOW_SIZE),
            "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
            "timeline_markers": list(timeline_marks),
            "timeline_total_ms": float((time.perf_counter() - t_request_start) * 1000.0),
        }

        decoder_batch_stats = {}
        if hasattr(self.snac_decoder, "get_batching_stats"):
            try:
                decoder_batch_stats = self.snac_decoder.get_batching_stats() or {}
            except Exception:
                decoder_batch_stats = {}
        if decoder_batch_stats:
            perf.update(decoder_batch_stats)

        self._apply_stats(perf, "llm_batch_wall_ms", llm_batch_wall_ms)
        self._apply_stats(perf, "llm_batch_gpu_ms", llm_batch_gpu_ms)
        self._apply_stats(perf, "llm_decode_wall_ms", llm_decode_wall_ms)
        self._apply_stats(perf, "llm_decode_gpu_ms", llm_decode_gpu_ms)
        self._apply_stats(perf, "tokens_per_batch", tokens_per_batch)
        self._apply_stats(perf, "llm_parse_step_ms", llm_parse_ms_samples)
        self._apply_stats(perf, "bicodec_decode_wall_ms", bicodec_decode_wall_ms)
        self._apply_stats(perf, "bicodec_decode_tokens", bicodec_decode_tokens)
        self._apply_stats(perf, "bicodec_decode_interval_ms", bicodec_decode_intervals_ms)
        self._apply_stats(perf, "decode_interval_applied_tokens", decode_interval_applied_tokens)
        self._apply_stats(perf, "decode_pending_requests", decode_pending_requests)

        perf["bicodec_decode_wall_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))
        perf["bicodec_decode_gpu_ms"] = 0.0
        perf["bicodec_decode_cpu_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))

        if total_generated_tokens > 0:
            perf["llm_time_per_token_ms"] = float(llm_generation_wall_ms) / float(total_generated_tokens)
        if llm_batch_wall_ms:
            perf["llm_time_per_batch_wall_ms"] = float(sum(llm_batch_wall_ms)) / float(len(llm_batch_wall_ms))
        if llm_batch_gpu_ms:
            perf["llm_time_per_batch_gpu_ms"] = float(sum(llm_batch_gpu_ms)) / float(len(llm_batch_gpu_ms))

        for mark in timeline_marks:
            stage = mark.get("stage")
            if not stage:
                continue
            perf[f"timeline_{stage}_ms"] = float(mark.get("t_ms", 0.0))
            perf[f"timeline_{stage}_delta_ms"] = float(mark.get("dt_ms", 0.0))

        tl_first_batch = perf.get("timeline_llm_first_batch_ms")
        tl_first_emit = perf.get("timeline_first_chunk_emitted_ms")
        tl_done = perf.get("timeline_request_done_ms")
        if isinstance(tl_first_batch, (int, float)):
            perf["timeline_to_first_batch_ms"] = float(tl_first_batch)
        if isinstance(tl_first_emit, (int, float)):
            perf["timeline_to_first_chunk_emitted_ms"] = float(tl_first_emit)
        if isinstance(tl_first_batch, (int, float)) and isinstance(tl_first_emit, (int, float)) and tl_first_emit >= tl_first_batch:
            perf["timeline_first_batch_to_first_chunk_emitted_ms"] = float(tl_first_emit - tl_first_batch)
        if isinstance(tl_first_emit, (int, float)) and isinstance(tl_done, (int, float)) and tl_done >= tl_first_emit:
            perf["timeline_first_chunk_to_request_done_ms"] = float(tl_done - tl_first_emit)

        sec_fields = {
            "time_in_queue": "llm_time_in_queue_ms",
            "scheduler_time": "llm_scheduler_ms",
            "model_forward_time": "llm_model_forward_ms",
            "model_execute_time": "llm_model_execute_ms",
            "first_token_latency": "llm_first_token_ms",
        }
        for src, dst in sec_fields.items():
            if src in final_request_metrics:
                perf[dst] = float(final_request_metrics[src]) * 1000.0

        def _delta_ms(start_s: Optional[float], end_s: Optional[float]) -> Optional[float]:
            if start_s is None or end_s is None:
                return None
            if end_s < start_s:
                return None
            return (end_s - start_s) * 1000.0

        queued_ts = final_request_metrics.get("queued_ts")
        scheduled_ts = final_request_metrics.get("scheduled_ts")
        first_token_ts = final_request_metrics.get("first_token_ts")
        last_token_ts = final_request_metrics.get("last_token_ts")
        q_to_sched = _delta_ms(queued_ts, scheduled_ts)
        sched_to_first = _delta_ms(scheduled_ts, first_token_ts)


Output:
Chunk ID: 5295b3
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 6050
Output:
        sched_to_first = _delta_ms(scheduled_ts, first_token_ts)
        first_to_last = _delta_ms(first_token_ts, last_token_ts)
        queued_to_last = _delta_ms(queued_ts, last_token_ts)
        if q_to_sched is not None:
            perf["llm_queued_to_scheduled_ms"] = q_to_sched
            perf["llm_time_in_queue_ms"] = q_to_sched
        if sched_to_first is not None:
            perf["llm_scheduled_to_first_token_ms"] = sched_to_first
            perf["llm_scheduler_ms"] = sched_to_first
        if first_to_last is not None:
            perf["llm_first_to_last_token_ms"] = first_to_last
            perf["llm_model_execute_ms"] = first_to_last
        if queued_to_last is not None:
            perf["llm_queued_to_last_token_ms"] = queued_to_last
            perf["llm_request_lifecycle_ms"] = queued_to_last

        logger.debug(
            "streaming complete audio=%.2fs ttfb=%.0fms chunks=%d llm_tokens=%d",
            perf["audio_duration_seconds"],
            perf["ttfb_ms"],
            perf["chunks_sent"],
            perf["llm_token_total"],
        )

        if emit_progress:
            yield (b"", perf)

    async def generate_speech_stream_indic_first_chunk(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[tuple, None]:
        """
        Generate speech for the FIRST chunk of multi-chunk text, capturing global tokens.

        This method is specifically for chunked generation: it yields (audio_bytes, global_ids)
        tuples instead of just audio_bytes. The caller captures global_ids from the first
        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.

        Use Case (chunked streaming with voice consistency):
            globals_captured = None
            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
                if globals_captured is None and global_ids:
                    globals_captured = global_ids  # Capture once
                yield audio_bytes
            # Now use globals_captured for subsequent chunks

        Args:
            speaker: Speaker name
            text: First text chunk to synthesize
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max tokens to generate
            repetition_penalty: Prevent repetition
            seed: Random seed for reproducibility

        Yields:
            Tuple of (audio_bytes, global_ids)
            - audio_bytes: Raw PCM audio (int16, 16kHz)
            - global_ids: List of 32 global token IDs (populated after first decode, else empty)

        Thread Safety:
            This method is stateless and thread-safe. Each call creates its own
            request-scoped state. No global state is modified or shared.
        """
        import time
        t_start = time.time()

        # Build prompt using Indic prompt builder
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,
            seed=seed,
            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
        )

        # BiCodec token buffers
        semantic_buffer = []
        global_buffer = []
        processed_token_count = 0  # Used for cumulative mode; ignored for delta mode.
        total_audio_chunks = 0

        # BiCodec streaming configuration
        EXPECTED_GLOBAL_COUNT = 32
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
        CROSSFADE_MS = 50

        # Generate unique request ID
        import uuid
        request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None
        captured_globals: List[int] = []  # Will be populated once we have 32 globals

        # Stream tokens with BiCodec extraction
        async for request_output in results_generator:
            generated_ids = request_output.outputs[0].token_ids

            # In DELTA mode token_ids are already incremental.
            if _STREAM_USE_DELTA_OUTPUT:
                new_token_ids = generated_ids
            else:
                new_token_ids = generated_ids[processed_token_count:]
                processed_token_count = len(generated_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)

            # Capture global tokens once we have all 32
            if not captured_globals and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                captured_globals = global_buffer[:EXPECTED_GLOBAL_COUNT]

            # Process if we have all global tokens
            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                semantic_count = len(semantic_buffer)
                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
                        last_decode_count = semantic_count

                        all_semantic = semantic_buffer
                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]

                        if self.snac_decoder.enable_batching:
                            audio_bytes = await self.snac_decoder.decode_single_async(
                                semantic_ids=all_semantic,
                                global_ids=window_global,
                                trim_warmup=False,
                                use_sliding_window=False
                            )
                        else:
                            audio_bytes = self.snac_decoder.decode_streaming(
                                semantic_ids=all_semantic,
                                global_ids=window_global,
                                use_sliding_window=False,
                                trim_warmup=False
                            )

                        if audio_bytes:
                            total_samples_decoded = len(audio_bytes) // 2

                            if total_samples_decoded > total_samples_emitted_to_user:
                                new_bytes_start = total_samples_emitted_to_user * 2
                                new_audio_bytes = audio_bytes[new_bytes_start:]

                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                    previous_chunk_tail,
                                    new_audio_bytes,
                                    sample_rate_hz=16000,
                                    crossfade_ms=CROSSFADE_MS,
                                )

                                samples_emitted_in_this_chunk = len(to_emit) // 2
                                total_samples_emitted_to_user += samples_emitted_in_this_chunk

                                if to_emit:
                                    total_audio_chunks += 1
                                    # Yield tuple: (audio_bytes, global_ids)
                                    yield (to_emit, captured_globals)

        # Final chunk processing
        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,
                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
                use_sliding_window=False,
                trim_warmup=False
            )
            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    samples_emitted_in_final = len(to_emit) // 2
                    total_samples_emitted_to_user += samples_emitted_in_final

                    if to_emit:
                        total_audio_chunks += 1
                        yield (to_emit, captured_globals)

        # Flush remaining tail
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            yield (previous_chunk_tail, captured_globals)

        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000
        logger.debug(
            "first-chunk complete audio=%.2fs chunks=%d captured_globals=%d",
            audio_duration_s,
            total_audio_chunks,
            len(captured_globals),
        )

    async def generate_speech_stream_indic_continuation(
        self,
        speaker: str,
        text: str,
        global_ids: List[int],
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        emit_progress: bool = False,
    ) -> AsyncGenerator[Union[bytes, Tuple[bytes, Dict[str, Any]]], None]:
        """
        Generate speech for CONTINUATION chunks using pre-captured global tokens.

        This is the key method for voice consistency in chunked generation:
        - Uses global_ids from the first chunk to maintain identical voice
        - Model generates only semantic tokens (skips global token generation)
        - Ensures no voice drift across chunks

        CRITICAL for production:
        - global_ids is request-scoped, passed explicitly (no shared state)
        - Thread-safe: each request has its own state
        - No global caching that could cause cross-request contamination

        Args:
            speaker: Speaker name (must match first chunk)
            text: Continuation text chunk to synthesize
            global_ids: 32 global token IDs captured from first chunk
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max tokens to generate
            repetition_penalty: Prevent repetition
            seed: Random seed (use same as first chunk for consistency)

        Yields:
            Audio bytes (int16 PCM, 16kHz mono)

        Raises:
            ValueError: If global_ids doesn't contain exactly 32 tokens

        Thread Safety:
            Fully thread-safe. All state is request-scoped and passed explicitly.
        """
        import time
        t_start = time.time()

        # Validate global tokens
        if len(global_ids) != _STREAM_EXPECTED_GLOBAL_COUNT:
            raise ValueError(
                f"Expected exactly {_STREAM_EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
                f"global_ids must be captured from first chunk generation."
            )

        # Build prompt WITH pre-filled global tokens
        # This tells the model to skip global generation and go straight to semantic tokens
        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)

        # Configure sampling
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,
            seed=seed,
            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
        )

        # BiCodec token buffers
        semantic_buffer = []
        global_buffer_unused = []  # Not used in continuation, but needed for parser API
        processed_token_count = 0  # Used for cumulative mode; ignored for delta mode.
        total_audio_chunks = 0
        total_generated_tokens = 0

        # BiCodec streaming configuration
        MIN_SEMANTIC_FOR_FIRST_CHUNK = _STREAM_MIN_SEMANTIC_FIRST_CHUNK
        DECODE_INTERVAL = _STREAM_DECODE_INTERVAL
        CROSSFADE_MS = _STREAM_CROSSFADE_MS
        WINDOW_SIZE = _STREAM_WINDOW_SIZE

        # Generate unique request ID
        import uuid
        request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None
        t_first_chunk_emitted: Optional[float] = None
        decode_calls = 0
        decode_wall_ms_total = 0.0
        decode_interval_applied_tokens: List[float] = []
        decode_pending_requests: List[float] = []

        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
        async for request_output in results_generator:
            generated_ids = request_output.outputs[0].token_ids

            # In DELTA mode token_ids are already incremental.
            if _STREAM_USE_DELTA_OUTPUT:
                new_token_ids = generated_ids
                processed_token_count += len(new_token_ids)
            else:
                new_token_ids = generated_ids[processed_token_count:]
                processed_token_count = len(generated_ids)
            total_generated_tokens += len(new_token_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)

            semantic_count = len(semantic_buffer)
            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                decode_interval, pending_requests = self._resolve_decode_interval(
                    first_chunk_emitted=t_first_chunk_emitted is not None
                )
                semantic_delta = semantic_count - last_decode_count
                if semantic_delta >= decode_interval:
                    decode_step_tokens = semantic_delta
                    last_decode_count = semantic_count
                    decode_interval_applied_tokens.append(float(decode_interval))
                    decode_pending_requests.append(float(pending_requests))
                    if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
                        decode_semantic = semantic_buffer[-WINDOW_SIZE:]
                    else:
                        decode_semantic = semantic_buffer

                    # Use pre-captured global tokens for decoding
                    decode_t0 = time.perf_counter()
                    if self.snac_decoder.enable_batching:
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=decode_semantic,
                            global_ids=global_ids,  # Use captured globals
                            trim_warmup=False,
                            use_sliding_window=False
                        )
                    else:
                        audio_bytes = self.snac_decoder.decode_streaming(
                            semantic_ids=decode_semantic,
                            global_ids=global_ids,  # Use captured globals
                            use_sliding_window=False,
                            trim_warmup=False
                        )
                    decode_calls += 1
                    decode_wall_ms_total += (time.perf_counter() - decode_t0) * 1000.0

                    if audio_bytes:
                        total_samples_decoded = len(audio_bytes) // 2
                        if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
                            new_samples_approx = decode_step_tokens * 320
                            if total_samples_decoded > new_samples_approx:
                                new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
                            else:
                                new_bytes_start = 0
                            new_audio_bytes = audio_bytes[new_bytes_start:]
                        elif total_samples_decoded > total_samples_emitted_to_user:
                            new_bytes_start = total_samples_emitted_to_user * 2
                            new_audio_bytes = audio_bytes[new_bytes_start:]
                        else:
                            new_audio_bytes = b""

                        if new_audio_bytes:
                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                previous_chunk_tail,
                                new_audio_bytes,
                                sample_rate_hz=16000,
                                crossfade_ms=CROSSFADE_MS,
                            )

                            if to_emit:
                                total_audio_chunks += 1
                                total_samples_emitted_to_user += len(to_emit) // 2
                                if t_first_chunk_emitted is None:
                                    t_first_chunk_emitted = time.perf_counter()
                                if emit_progress:
                                    yield (to_emit, {})
                                else:
                                    yield to_emit

        # Final chunk processing
        if len(semantic_buffer) > last_decode_count:
            decode_t0 = time.perf_counter()
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,
                global_ids=global_ids,  # Use captured globals
                use_sliding_window=False,
                trim_warmup=False
            )
            decode_calls += 1
            decode_wall_ms_total += (time.perf_counter() - decode_t0) * 1000.0
            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    if to_emit:
                        total_audio_chunks += 1
                        total_samples_emitted_to_user += len(to_emit) // 2
                        if t_first_chunk_emitted is None:
                            t_first_chunk_emitted = time.perf_counter()
                        if emit_progress:
                            yield (to_emit, {})
                        else:
                            yield to_emit

        # Flush remaining tail
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            if t_first_chunk_emitted is None:
                t_first_chunk_emitted = time.perf_counter()
            if emit_progress:
                yield (previous_chunk_tail, {})
            else:
                yield previous_chunk_tail

        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000
        logger.debug(
            "continuation complete audio=%.2fs chunks=%d",
            audio_duration_s,
            total_audio_chunks,
        )

        if emit_progress:
            perf: Dict[str, Any] = {
                "llm_token_total": int(total_generated_tokens),
                "semantic_token_total": int(len(semantic_buffer)),
                "global_token_total": int(len(global_ids)),
                "bicodec_decode_calls": int(decode_calls),
                "llm_decode_calls": int(decode_calls),
                "bicodec_decode_wall_ms": float(decode_wall_ms_total),
                "bicodec_decode_gpu_ms": 0.0,
                "bicodec_decode_cpu_ms": float(decode_wall_ms_total),
                "chunks_sent": int(total_audio_chunks),
                "audio_duration_seconds": float(audio_duration_s),
                "ttfb_ms": float((t_first_chunk_emitted - t_start) * 1000.0 if t_first_chunk_emitted else 0.0),
                "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
                "stream_decode_interval_tokens": int(DECODE_INTERVAL),
                "stream_decode_interval_first_tokens": int(_STREAM_DECODE_INTERVAL_FIRST),
                "stream_decode_interval_busy_tokens": int(_STREAM_DECODE_INTERVAL_BUSY),
                "stream_decode_busy_pending_threshold": int(_STREAM_DECODE_BUSY_PENDING),
                "stream_adaptive_decode": bool(_STREAM_ADAPTIVE_DECODE),
                "stream_window_size_tokens": int(WINDOW_SIZE),
                "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
            }
            self._apply_stats(perf, "decode_interval_applied_tokens", decode_interval_applied_tokens)
            self._apply_stats(perf, "decode_pending_requests", decode_pending_requests)
            decoder_batch_stats = {}
            if hasattr(self.snac_decoder, "get_batching_stats"):
                try:
                    decoder_batch_stats = self.snac_decoder.get_batching_stats() or {}
                except Exception:
                    decoder_batch_stats = {}
            if decoder_batch_stats:
                perf.update(decoder_batch_stats)
            yield (b"", perf)


**Designing async admission control**


sed -n '1,140p' veena3modal/services/tts_runtime.py


Output:
Chunk ID: 6692fa
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 1268
Output:
"""
TTS runtime wrapper (container-scoped singleton).

In Modal, each container should load the model once and then handle many requests.
This wrapper owns:
- SparkTTSModel (vLLM engine)
- IndicPromptBuilder
- BiCodecDecoder
- SparkTTSPipeline + Veena3SlidingWindowPipeline
- optional SuperResolutionService

Imports framework-agnostic inference + processing code from `veena3modal/core`,
`veena3modal/processing`, and `veena3modal/audio`.
"""

from __future__ import annotations

import os
import sys
import time
import logging
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

logger = logging.getLogger(__name__)

@dataclass
class TTSRuntime:
    """
    Holds long-lived, per-container inference objects.

    Thread Safety:
        This class is designed for async concurrency within a single container.
        The vLLM engine handles internal batching and scheduling.
        Do NOT share across processes without proper synchronization.
    """
    model: Any = None
    pipeline: Any = None
    streaming_pipeline: Any = None
    long_text_processor: Any = None
    prompt_builder: Any = None
    bicodec_decoder: Any = None
    sr_service: Optional[Any] = None

    model_version: str = "not_loaded"
    is_loaded: bool = False
    load_time_ms: float = 0.0

    # Configuration
    model_path: str = ""
    bicodec_path: str = ""
    sr_checkpoint_dir: Optional[str] = None
    device: str = "cuda"

    # OPTIMIZATION: Pre-computed global tokens per speaker (eliminates ~110ms pre-roll in streaming)
    # Maps speaker_name -> list of 32 global token IDs captured at startup
    speaker_global_cache: Dict[str, List[int]] = field(default_factory=dict)

# Module-level singleton (per-container)
_runtime: Optional[TTSRuntime] = None

def get_runtime() -> Optional[TTSRuntime]:
    """Get the current TTS runtime singleton."""
    return _runtime

def is_initialized() -> bool:
    """Check if the runtime is initialized and ready."""
    return _runtime is not None and _runtime.is_loaded

def initialize_runtime(
    model_path: Optional[str] = None,
    bicodec_path: Optional[str] = None,
    sr_checkpoint_dir: Optional[str] = None,
    device: str = "cuda",
    hf_token: Optional[str] = None,
    gpu_memory_utilization: float = 0.25,
    enable_sr: bool = False,
    num_engines: int = 1,
    max_num_batched_tokens: Optional[int] = None,
    max_num_seqs: Optional[int] = None,
    enable_chunked_prefill: Optional[bool] = None,
    enable_prefix_caching: Optional[bool] = None,
    disable_log_stats: Optional[bool] = None,
    enforce_eager: Optional[bool] = None,
    precompute_speaker_globals: Optional[bool] = None,
) -> TTSRuntime:
    """
    Initialize the TTS runtime with all components.

    This should be called once per container (e.g., in Modal's @modal.enter).

    Args:
        model_path: Path to Spark TTS model (env: SPARK_TTS_MODEL_PATH)
        bicodec_path: Path to BiCodec model (env: BICODEC_MODEL_PATH, defaults to model_path)
        sr_checkpoint_dir: Path to super-resolution checkpoints (env: AP_BWE_CHECKPOINT_DIR)
        device: Device for inference (cuda/cpu)
        hf_token: HuggingFace token for private models (env: HF_TOKEN)
        gpu_memory_utilization: vLLM GPU memory fraction (default: 0.25)
        enable_sr: Enable super-resolution service
        num_engines: Number of vLLM engine instances (default: 1, set to 2-3 for Tier 3 optimization)
        max_num_batched_tokens: Optional vLLM scheduler cap override.
        max_num_seqs: Optional vLLM concurrent sequence cap override.
        enable_chunked_prefill: Optional vLLM chunked prefill toggle.
        enable_prefix_caching: Optional vLLM prefix caching toggle.
        disable_log_stats: Optional vLLM internal stats log toggle.
        enforce_eager: Optional vLLM eager-mode toggle (disables CUDA graphs when true).
        precompute_speaker_globals: Whether to warm speaker global tokens at startup.
            If None, reads PRECOMPUTE_SPEAKER_GLOBALS env var (default: false).

    Returns:
        Initialized TTSRuntime instance

    Raises:
        RuntimeError: If initialization fails
    """
    global _runtime

    start_time = time.time()

    # Resolve paths from env vars if not provided
    model_path = model_path or os.environ.get(
        'SPARK_TTS_MODEL_PATH',
        os.environ.get('MODEL_PATH', '/models/spark_tts_4speaker')
    )
    bicodec_path = bicodec_path or os.environ.get(
        'BICODEC_MODEL_PATH',
        model_path  # BiCodec is usually in the same directory
    )
    sr_checkpoint_dir = sr_checkpoint_dir or os.environ.get('AP_BWE_CHECKPOINT_DIR')
    hf_token = hf_token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')
    if precompute_speaker_globals is None:
        raw_flag = os.environ.get("PRECOMPUTE_SPEAKER_GLOBALS", "false").strip().lower()
        precompute_speaker_globals = raw_flag in {"1", "true", "yes", "on"}

    logger.info(f"Initializing TTS runtime...")


**Designing streaming admission controller**


I’m now implementing a shared streaming admission controller in `tts_runtime` (max inflight, bounded queue, wait budget, overload errors, and per-request wait metrics). After that I’ll hook HTTP/WS entrypoints to pre-acquire slots so overload returns cleanly (429/WS overload), then run fresh stress sweeps.


Input:
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
 import os
 import sys
 import time
 import logging
+import threading
+import asyncio
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
@@
 logger = logging.getLogger(__name__)
+
+
+def _env_int(name: str, default: int, minimum: int = 0) -> int:
+    raw = os.environ.get(name, "")
+    if not raw:
+        return default
+    try:
+        value = int(raw)
+    except ValueError:
+        return default
+    return max(minimum, value)
+
+
+def _env_float(name: str, default: float, minimum: float = 0.0) -> float:
+    raw = os.environ.get(name, "")
+    if not raw:
+        return default
+    try:
+        value = float(raw)
+    except ValueError:
+        return default
+    return max(minimum, value)
+
+
+@dataclass
+class StreamingAdmissionLease:
+    """Admission ticket for one active streaming request."""
+
+    wait_ms: float = 0.0
+    queued: bool = False
+    inflight_on_grant: int = 0
+    waiters_on_grant: int = 0
+    queue_depth_on_entry: int = 0
+    released: bool = False
+
+
+class StreamingAdmissionError(RuntimeError):
+    """Raised when streaming admission control cannot accept a request."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        reason: str,
+        retry_after_ms: float = 0.0,
+        snapshot: Optional[Dict[str, float]] = None,
+    ):
+        super().__init__(message)
+        self.reason = reason
+        self.retry_after_ms = float(max(0.0, retry_after_ms))
+        self.snapshot = snapshot or {}
+
+
+class StreamingAdmissionController:
+    """
+    Lightweight admission controller for streaming requests.
+
+    Keeps a hard cap on active streams (`max_inflight`) and an optional bounded
+    wait queue (`max_queue`) with per-request wait timeout (`max_wait_ms`).
+    """
+
+    def __init__(
+        self,
+        max_inflight: int,
+        max_queue: int,
+        max_wait_ms: float,
+        poll_ms: float,
+    ):
+        self.max_inflight = int(max(0, max_inflight))
+        self.max_queue = int(max(0, max_queue))
+        self.max_wait_ms = float(max(0.0, max_wait_ms))
+        self.poll_ms = float(max(0.5, poll_ms))
+        self.enabled = self.max_inflight > 0
+
+        self._lock = threading.Lock()
+        self._inflight = 0
+        self._waiters = 0
+        self._total_admitted = 0
+        self._total_queue_entries = 0
+        self._total_queue_wait_ms = 0.0
+        self._total_rejected_queue_full = 0
+        self._total_rejected_timeout = 0
+
+    @classmethod
+    def from_env(cls) -> "StreamingAdmissionController":
+        return cls(
+            max_inflight=_env_int("VEENA3_STREAM_ADMISSION_MAX_INFLIGHT", 0, minimum=0),
+            max_queue=_env_int("VEENA3_STREAM_ADMISSION_MAX_QUEUE", 0, minimum=0),
+            max_wait_ms=_env_float("VEENA3_STREAM_ADMISSION_MAX_WAIT_MS", 0.0, minimum=0.0),
+            poll_ms=_env_float("VEENA3_STREAM_ADMISSION_POLL_MS", 2.0, minimum=0.5),
+        )
+
+    def _snapshot_locked(self) -> Dict[str, float]:
+        queue_avg = (
+            self._total_queue_wait_ms / self._total_queue_entries
+            if self._total_queue_entries > 0
+            else 0.0
+        )
+        return {
+            "enabled": 1.0 if self.enabled else 0.0,
+            "max_inflight": float(self.max_inflight),
+            "max_queue": float(self.max_queue),
+            "max_wait_ms": float(self.max_wait_ms),
+            "poll_ms": float(self.poll_ms),
+            "inflight_now": float(self._inflight),
+            "waiters_now": float(self._waiters),
+            "admitted_total": float(self._total_admitted),
+            "queue_entries_total": float(self._total_queue_entries),
+            "queue_wait_ms_avg": float(queue_avg),
+            "rejected_queue_full_total": float(self._total_rejected_queue_full),
+            "rejected_timeout_total": float(self._total_rejected_timeout),
+        }
+
+    def snapshot(self) -> Dict[str, float]:
+        with self._lock:
+            return self._snapshot_locked()
+
+    async def acquire(self) -> StreamingAdmissionLease:
+        start = time.perf_counter()
+        queued = False
+        queue_depth_on_entry = 0
+        deadline = (
+            start + (self.max_wait_ms / 1000.0)
+            if self.max_wait_ms > 0.0
+            else None
+        )
+
+        while True:
+            now = time.perf_counter()
+            with self._lock:
+                if (not self.enabled) or (self._inflight < self.max_inflight):
+                    wait_ms = (now - start) * 1000.0
+                    self._inflight += 1
+                    if queued:
+                        self._waiters = max(0, self._waiters - 1)
+                        self._total_queue_wait_ms += max(0.0, wait_ms)
+                    self._total_admitted += 1
+                    return StreamingAdmissionLease(
+                        wait_ms=float(wait_ms),
+                        queued=bool(queued),
+                        inflight_on_grant=int(self._inflight),
+                        waiters_on_grant=int(self._waiters),
+                        queue_depth_on_entry=int(queue_depth_on_entry),
+                    )
+
+                if not queued:
+                    if self._waiters >= self.max_queue:
+                        self._total_rejected_queue_full += 1
+                        snapshot = self._snapshot_locked()
+                        raise StreamingAdmissionError(
+                            "streaming admission queue full",
+                            reason="queue_full",
+                            retry_after_ms=self.poll_ms,
+                            snapshot=snapshot,
+                        )
+                    queued = True
+                    self._waiters += 1
+                    queue_depth_on_entry = self._waiters
+                    self._total_queue_entries += 1
+
+            if deadline is not None and now >= deadline:
+                with self._lock:
+                    if queued:
+                        self._waiters = max(0, self._waiters - 1)
+                    self._total_rejected_timeout += 1
+                    snapshot = self._snapshot_locked()
+                raise StreamingAdmissionError(
+                    "streaming admission wait timeout",
+                    reason="wait_timeout",
+                    retry_after_ms=self.poll_ms,
+                    snapshot=snapshot,
+                )
+
+            await asyncio.sleep(self.poll_ms / 1000.0)
+
+    def release(self, lease: Optional[StreamingAdmissionLease]) -> None:
+        if lease is None or lease.released:
+            return
+        lease.released = True
+        with self._lock:
+            self._inflight = max(0, self._inflight - 1)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Adding field to dataclass**


Input:
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
     # OPTIMIZATION: Pre-computed global tokens per speaker (eliminates ~110ms pre-roll in streaming)
     # Maps speaker_name -> list of 32 global token IDs captured at startup
     speaker_global_cache: Dict[str, List[int]] = field(default_factory=dict)
+    stream_admission: Optional[StreamingAdmissionController] = None
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Planning helper function additions**


Input:
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
 def is_initialized() -> bool:
     """Check if the runtime is initialized and ready."""
     return _runtime is not None and _runtime.is_loaded
+
+
+def _get_stream_admission_controller(runtime: Optional[TTSRuntime]) -> StreamingAdmissionController:
+    """Return initialized streaming admission controller for this runtime."""
+    if runtime is None:
+        raise RuntimeError("TTS runtime not initialized")
+    controller = runtime.stream_admission
+    if controller is None:
+        controller = StreamingAdmissionController.from_env()
+        runtime.stream_admission = controller
+    return controller
+
+
+def get_streaming_admission_snapshot() -> Dict[str, float]:
+    """
+    Return current admission state + cumulative counters.
+
+    Safe to call anytime; when runtime is not initialized returns an empty dict.
+    """
+    runtime = get_runtime()
+    if runtime is None:
+        return {}
+    return _get_stream_admission_controller(runtime).snapshot()
+
+
+async def acquire_streaming_slot() -> StreamingAdmissionLease:
+    """Acquire one streaming admission slot or raise StreamingAdmissionError."""
+    if not is_initialized():
+        raise RuntimeError("TTS runtime not initialized")
+    runtime = get_runtime()
+    controller = _get_stream_admission_controller(runtime)
+    return await controller.acquire()
+
+
+def release_streaming_slot(lease: Optional[StreamingAdmissionLease]) -> None:
+    """Release a previously acquired streaming slot (idempotent)."""
+    runtime = get_runtime()
+    if runtime is None:
+        return
+    controller = _get_stream_admission_controller(runtime)
+    controller.release(lease)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Adding runtime instantiation patch**