Output:
Chunk ID: a0a673
Wall time: 0.0537 seconds
Process exited with code 0
Original token count: 2084
Output:
"""
Prometheus metrics for Veena3 TTS service.

Features:
- Request counters (by speaker, format, status)
- TTFB histogram
- RTF histogram
- Audio duration histogram
- Model load time gauge

Usage:
    from veena3modal.shared.metrics import record_request_received, record_ttfb

    record_request_received(speaker="lipakshi", stream=True, format="wav")
    record_ttfb(ttfb_seconds=0.25, speaker="lipakshi", stream=True)
"""

import re
from typing import Optional

# Try to import prometheus_client, fall back to no-op if not installed
try:
    from prometheus_client import (
        Counter, Histogram, Gauge,
        CollectorRegistry, generate_latest, CONTENT_TYPE_LATEST
    )
    PROMETHEUS_AVAILABLE = True
except ImportError:
    PROMETHEUS_AVAILABLE = False

# Module-level registry (singleton)
_registry: Optional['CollectorRegistry'] = None

# Metrics instances (lazy initialized)
_requests_total: Optional['Counter'] = None
_requests_completed: Optional['Counter'] = None
_requests_failed: Optional['Counter'] = None
_ttfb_seconds: Optional['Histogram'] = None
_rtf: Optional['Histogram'] = None
_audio_duration_seconds: Optional['Histogram'] = None
_chunks_sent: Optional['Histogram'] = None
_model_load_seconds: Optional['Gauge'] = None
_model_loaded: Optional['Gauge'] = None

def get_metrics_registry() -> Optional['CollectorRegistry']:
    """Get or create the metrics registry."""
    global _registry

    if not PROMETHEUS_AVAILABLE:
        return None

    if _registry is None:
        _registry = CollectorRegistry()
        _initialize_metrics()

    return _registry

def _initialize_metrics() -> None:
    """Initialize all metrics with the registry."""
    global _requests_total, _requests_completed, _requests_failed
    global _ttfb_seconds, _rtf, _audio_duration_seconds, _chunks_sent
    global _model_load_seconds, _model_loaded

    if not PROMETHEUS_AVAILABLE or _registry is None:
        return

    # Request counters
    _requests_total = Counter(
        'veena3_tts_requests_total',
        'Total TTS requests received',
        ['speaker', 'stream', 'format'],
        registry=_registry
    )

    _requests_completed = Counter(
        'veena3_tts_requests_completed_total',
        'Total TTS requests completed successfully',
        ['speaker', 'stream', 'status_code'],
        registry=_registry
    )

    _requests_failed = Counter(
        'veena3_tts_requests_failed_total',
        'Total TTS requests failed',
        ['speaker', 'error_code', 'status_code'],
        registry=_registry
    )

    # Performance histograms
    _ttfb_seconds = Histogram(
        'veena3_tts_ttfb_seconds',
        'Time to first byte in seconds',
        ['speaker', 'stream'],
        buckets=(0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0),
        registry=_registry
    )

    _rtf = Histogram(
        'veena3_tts_rtf',
        'Real-time factor (generation_time / audio_duration)',
        ['speaker'],
        buckets=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.5, 2.0),
        registry=_registry
    )

    _audio_duration_seconds = Histogram(
        'veena3_tts_audio_duration_seconds',
        'Generated audio duration in seconds',
        ['speaker'],
        buckets=(1, 2, 5, 10, 20, 30, 60, 120, 300),
        registry=_registry
    )

    _chunks_sent = Histogram(
        'veena3_tts_chunks_sent',
        'Number of audio chunks sent in streaming mode',
        ['speaker'],
        buckets=(1, 2, 5, 10, 20, 50, 100),
        registry=_registry
    )

    # Model metrics
    _model_load_seconds = Gauge(
        'veena3_tts_model_load_seconds',
        'Time taken to load the model',
        ['model_version'],
        registry=_registry
    )

    _model_loaded = Gauge(
        'veena3_tts_model_loaded',
        'Whether the model is loaded (1) or not (0)',
        ['model_version'],
        registry=_registry
    )

def sanitize_label(value: str) -> str:
    """
    Sanitize a string for use as a Prometheus label.

    Prometheus labels must match [a-zA-Z_][a-zA-Z0-9_]*
    """
    # Replace non-alphanumeric chars (except underscore) with underscore
    sanitized = re.sub(r'[^a-zA-Z0-9_]', '_', str(value))
    # Ensure doesn't start with a digit
    if sanitized and sanitized[0].isdigit():
        sanitized = '_' + sanitized
    return sanitized

def record_request_received(
    speaker: str,
    stream: bool,
    format: str,
) -> None:
    """Record a request received."""
    if _requests_total is None:
        get_metrics_registry()  # Initialize if needed

    if _requests_total is not None:
        _requests_total.labels(
            speaker=sanitize_label(speaker),
            stream=str(stream).lower(),
            format=sanitize_label(format),
        ).inc()

def record_request_completed(
    status_code: int,
    duration_seconds: float,
    speaker: str,
    stream: bool,
) -> None:
    """Record a successful request completion."""
    if _requests_completed is None:
        get_metrics_registry()

    if _requests_completed is not None:
        _requests_completed.labels(
            speaker=sanitize_label(speaker),
            stream=str(stream).lower(),
            status_code=str(status_code),
        ).inc()

def record_request_failed(
    status_code: int,
    error_code: str,
    speaker: str,
) -> None:
    """Record a failed request."""
    if _requests_failed is None:
        get_metrics_registry()

    if _requests_failed is not None:
        _requests_failed.labels(
            speaker=sanitize_label(speaker),
            error_code=sanitize_label(error_code),
            status_code=str(status_code),
        ).inc()

def record_ttfb(
    ttfb_seconds: float,
    speaker: str,
    stream: bool,
) -> None:
    """Record time to first byte."""
    if _ttfb_seconds is None:
        get_metrics_registry()

    if _ttfb_seconds is not None:
        _ttfb_seconds.labels(
            speaker=sanitize_label(speaker),
            stream=str(stream).lower(),
        ).observe(ttfb_seconds)

def record_rtf(
    rtf: float,
    speaker: str,
) -> None:
    """Record real-time factor."""
    if _rtf is None:
        get_metrics_registry()

    if _rtf is not None:
        _rtf.labels(
            speaker=sanitize_label(speaker),
        ).observe(rtf)

def record_audio_duration(
    duration_seconds: float,
    speaker: str,
) -> None:
    """Record generated audio duration."""
    if _audio_duration_seconds is None:
        get_metrics_registry()

    if _audio_duration_seconds is not None:
        _audio_duration_seconds.labels(
            speaker=sanitize_label(speaker),
        ).observe(duration_seconds)

def record_chunks_sent(
    chunks: int,
    speaker: str,
) -> None:
    """Record number of chunks sent in streaming."""
    if _chunks_sent is None:
        get_metrics_registry()

    if _chunks_sent is not None:
        _chunks_sent.labels(
            speaker=sanitize_label(speaker),
        ).observe(chunks)

def record_model_load_time(
    duration_seconds: float,
    model_version: str,
) -> None:
    """Record model loading time."""
    if _model_load_seconds is None:
        get_metrics_registry()

    if _model_load_seconds is not None:
        _model_load_seconds.labels(
            model_version=sanitize_label(model_version),
        ).set(duration_seconds)

def set_model_loaded(
    loaded: bool,
    model_version: str,
) -> None:
    """Set model loaded status."""
    if _model_loaded is None:
        get_metrics_registry()

    if _model_loaded is not None:
        _model_loaded.labels(
            model_version=sanitize_label(model_version),
        ).set(1 if loaded else 0)

def get_metrics_text() -> str:
    """
    Get metrics in Prometheus text format.

    Returns:
        Prometheus text format metrics string
    """
    registry = get_metrics_registry()

    if not PROMETHEUS_AVAILABLE or registry is None:
        return "# Prometheus client not available\n"

    return generate_latest(registry).decode('utf-8')

def get_content_type() -> str:
    """Get the content type for Prometheus metrics."""
    if PROMETHEUS_AVAILABLE:
        return CONTENT_TYPE_LATEST
    return "text/plain; charset=utf-8"


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '1,260p' veena3modal/api/schemas.py


Output:
Chunk ID: be5744
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 1906
Output:
"""
Pydantic schemas for TTS API requests/responses.

Ported from veena3srv/apps/api/serializers.py with full validation parity.
No Django dependencies - pure Pydantic for Modal deployment.
"""

from __future__ import annotations

import re
from enum import Enum
from typing import Any, Dict, List, Optional, Union
from uuid import UUID

from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator

# === Constants (mirrored from veena3srv/apps/inference/constants.py) ===
# These are duplicated here to avoid Django import chain; must stay in sync.

MAX_TEXT_LENGTH = 50000

# Spark TTS Speaker System - 12 Speakers
# Model: BayAreaBoys/spark_tts_4speaker (HuggingFace) has 12 speaker tokens
SPEAKER_MAP = {
    "lipakshi": 0, "vardan": 1, "reet": 2, "Nandini": 3,
    "krishna": 4, "anika": 5, "adarsh": 6, "Nilay": 7,
    "Aarvi": 8, "Asha": 9, "Bittu": 10, "Mira": 11,
}

# Friendly speaker name mappings (user-facing → internal)
FRIENDLY_SPEAKER_MAP = {
    "Mitra": "lipakshi", "Aaranya": "reet", "Taru": "Nandini",
    "Neer": "Nilay", "Dhruva": "vardan", "Ira": "anika",
    "Veda": "adarsh", "Aria": "krishna",
    "Aarvi": "Aarvi", "Asha": "Asha", "Bittu": "Bittu", "Mira": "Mira",
}

INDIC_SPEAKERS = list(SPEAKER_MAP.keys())
ALL_SPEAKER_NAMES = INDIC_SPEAKERS + list(FRIENDLY_SPEAKER_MAP.keys())

# Emotion tags (Spark TTS bracket format)
INDIC_EMOTION_TAGS = [
    "[angry]", "[curious]", "[excited]", "[giggle]", "[laughs harder]",
    "[laughs]", "[screams]", "[sighs]", "[sings]", "[whispers]"
]

# Legacy emotion tag mapping (<angle> → [bracket])
LEGACY_EMOTION_MAP = {
    "<angry>": "[angry]", "<curious>": "[curious]", "<excited>": "[excited]",
    "<giggle>": "[giggle]", "<laugh_harder>": "[laughs harder]",
    "<laugh>": "[laughs]", "<scream>": "[screams]", "<sigh>": "[sighs]",
    "<sing>": "[sings]", "<whisper>": "[whispers]"
}

# === Helper Functions ===

def resolve_speaker_name(name: str) -> str:
    """
    Resolve friendly speaker name to internal name.

    Examples:
        resolve_speaker_name("Mitra") -> "lipakshi"
        resolve_speaker_name("lipakshi") -> "lipakshi"

    Raises:
        ValueError: if name is not a valid speaker name
    """
    if name in FRIENDLY_SPEAKER_MAP:
        return FRIENDLY_SPEAKER_MAP[name]
    if name in INDIC_SPEAKERS:
        return name
    raise ValueError(
        f"Invalid speaker name: {name}. Valid names: {', '.join(ALL_SPEAKER_NAMES)}"
    )

# Emotion normalization mapping (various formats → [emotion])
_EMOTION_TO_TAG = {
    **{k.strip('<>'): v for k, v in LEGACY_EMOTION_MAP.items()},
    'laughing': '[laughs]', 'laugh': '[laughs]', 'laughs harder': '[laughs harder]',
    'laugh harder': '[laughs harder]', 'laugh_harder': '[laughs harder]',
    'sighs': '[sighs]', 'sigh': '[sighs]', 'giggles': '[giggle]', 'giggle': '[giggle]',
    'angry': '[angry]', 'excited': '[excited]', 'whispers': '[whispers]',
    'whisper': '[whispers]', 'screaming': '[screams]', 'scream': '[screams]',
    'screams': '[screams]', 'singing': '[sings]', 'sing': '[sings]',
    'sings': '[sings]', 'curious': '[curious]',
}

def normalize_emotion_tags(text: str) -> str:
    """
    Normalize emotion tags to Spark TTS bracket format.

    Conversions:
    - <laugh> → [laughs]
    - [laughing] → [laughs]
    - etc.
    """
    # Convert old <emotion> format to [emotion]
    angle_pattern = r'<([a-z_]+)>'

    def replace_angle(match):
        emotion = match.group(1).lower().strip()
        if emotion in _EMOTION_TO_TAG:
            return _EMOTION_TO_TAG[emotion]
        old_tag = f'<{emotion}>'
        if old_tag in LEGACY_EMOTION_MAP:
            return LEGACY_EMOTION_MAP[old_tag]
        return f'[{emotion}]'  # Unknown, convert to bracket anyway

    text = re.sub(angle_pattern, replace_angle, text)

    # Normalize [emotion] variants
    bracket_pattern = r'\[([^\]]+)\]'

    def replace_bracket(match):
        emotion = match.group(1).lower().strip()
        bracket_tag = f'[{emotion}]'
        if bracket_tag in INDIC_EMOTION_TAGS:
            return bracket_tag
        if emotion in _EMOTION_TO_TAG:
            return _EMOTION_TO_TAG[emotion]
        # Try singular/plural
        if emotion.endswith('s') and emotion[:-1] in _EMOTION_TO_TAG:
            return _EMOTION_TO_TAG[emotion[:-1]]
        if (emotion + 's') in _EMOTION_TO_TAG:
            return _EMOTION_TO_TAG[emotion + 's']
        return bracket_tag  # Keep as-is

    return re.sub(bracket_pattern, replace_bracket, text)

# === Enums ===

class AudioFormat(str, Enum):
    """Supported audio output formats."""
    WAV = "wav"
    OPUS = "opus"
    MP3 = "mp3"
    MULAW = "mulaw"
    FLAC = "flac"

class OutputSampleRate(str, Enum):
    """Output sample rate options."""
    SR_16KHZ = "16khz"
    SR_48KHZ = "48khz"

# === Request Schema ===

class TTSGenerateRequest(BaseModel):
    """
    Request schema for /v1/tts/generate.

    Mirrors validation rules from veena3srv/apps/api/serializers.py.
    """

    # Required
    text: str = Field(
        ...,
        min_length=1,
        max_length=MAX_TEXT_LENGTH,
        description="Text to synthesize (max 50,000 characters)"
    )

    # Speaker (required for indic_speakers model)
    speaker: Optional[str] = Field(
        None,
        max_length=50,
        description="Speaker name (internal or friendly)"
    )

    # Voice design (not currently supported)
    description: Optional[str] = Field(
        None,
        max_length=2000,
        description="Voice description (for voiceDesign model, NOT currently supported)"
    )
    voice_id: Optional[UUID] = Field(
        None,
        description="UUID of pre-created voice profile (for voiceDesign model)"
    )

    # Generation options
    model: Optional[str] = Field(
        None,
        max_length=50,
        description="Model to use (optional). Default: indic_speakers"
    )
    seed: Optional[int] = Field(
        None,
        ge=0,
        le=2**31 - 1,
        description="Random seed for reproducibility (0-2147483647)"
    )
    stream: bool = Field(
        False,
        description="Stream audio as it's generated"
    )
    format: AudioFormat = Field(
        AudioFormat.WAV,
        description="Audio output format"
    )
    sample_rate: int = Field(
        16000,
        description="Sample rate in Hz (default: 16000)"
    )

    # Advanced parameters
    temperature: float = Field(
        0.4,
        ge=0.0,
        le=2.0,
        description="Sampling temperature (0.0-2.0)"
    )
    top_k: int = Field(
        50,
        ge=1,
        le=100,
        description="Top-k sampling (1-100)"
    )
    top_p: float = Field(
        1.0,
        ge=0.0,
        le=1.0,
        description="Nucleus sampling (0.0-1.0)"
    )
    max_tokens: int = Field(
        4096,
        ge=128,
        le=4096,
        description="Maximum BiCodec tokens to generate"
    )
    repetition_penalty: float = Field(
        1.05,
        ge=1.0,
        le=2.0,
        description="Repetition penalty (1.0-2.0)"
    )

    # Preprocessing toggles
    normalize: bool = Field(
        True,
        description="Apply text normalization before TTS"
    )
    normalize_verbose: bool = Field(
        False,
        description="Return normalized text in X-Normalized-Text header"
    )
    chunking: bool = Field(
        True,
        description="Enable intelligent text chunking for long inputs"
    )

    # Output options
    output: OutputSampleRate = Field(
        OutputSampleRate.SR_16KHZ,


Summary: auto


Output:
Chunk ID: 3adff3
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 39
Output:
"""Allow running as: python -m veena3modal (shortcut for local_server)."""
from veena3modal.local_server import main

if __name__ == "__main__":
    main()


I found most of the existing paths that affect latency. I’ll now add opt-in structured instrumentation (per request + per stage) and a benchmark driver that runs 1-RPS to high-concurrency synthetic loads directly against the local FastAPI endpoint.


sed -n '1,340p' veena3modal/core/bicodec_decoder.py


Output:
Chunk ID: 1ea3b9
Wall time: 0.0517 seconds
Process exited with code 0
Original token count: 3121
Output:
"""
Spark TTS BiCodec Decoder

Decodes BiCodec audio tokens (semantic + global) to audio waveforms.
Replaces SNAC decoder for Spark TTS model.
"""

import torch
import numpy as np
import sys
from pathlib import Path
from typing import List, Optional, Tuple

# Ensure `sparktts` (vendored under external/) is importable in local/dev runs.
# In Modal, PYTHONPATH already includes `/root/external/sparktts`.
try:
    from sparktts.models.audio_tokenizer import BiCodecTokenizer
except ImportError:  # pragma: no cover - environment-dependent
    _this_file = Path(__file__).resolve()
    _sparktts_path = None
    for _p in [_this_file.parent] + list(_this_file.parents):
        _candidate = _p / "external" / "sparktts"
        if _candidate.is_dir():
            _sparktts_path = str(_candidate)
            break
    if _sparktts_path and _sparktts_path not in sys.path:
        sys.path.insert(0, _sparktts_path)
    from sparktts.models.audio_tokenizer import BiCodecTokenizer

from veena3modal.core.constants import (
    BICODEC_TOKENIZER_PATH,
    AUDIO_SAMPLE_RATE,
)

class BiCodecDecoder:
    """
    BiCodec Decoder for Spark TTS.

    Decodes semantic and global tokens to audio waveforms using BiCodec.
    This replaces the SNAC decoder used in the old Indic Orpheus model.

    Supports streaming with sliding window approach (like SNAC).
    """

    def __init__(
        self,
        device: str = "cuda",
        model_path: str = BICODEC_TOKENIZER_PATH,
        enable_batching: bool = False,  # For compatibility with streaming pipeline
    ):
        """
        Initialize BiCodec decoder.

        Args:
            device: Device for BiCodec model (cuda/cpu)
            model_path: Path to BiCodec model checkpoint
            enable_batching: Enable async batching for streaming (not used yet)
        """
        self.device = torch.device(device if torch.cuda.is_available() else "cpu")
        self.model_path = model_path
        self.enable_batching = enable_batching  # For streaming pipeline compatibility

        print(f"🎵 Loading BiCodec audio tokenizer from {model_path}...")
        print(f"   Device: {self.device}")

        # Initialize BiCodec tokenizer
        self.audio_tokenizer = BiCodecTokenizer(model_path, device=str(self.device))

        # OPTIMIZATION: Ensure model is on device once at init, not per-decode call
        self.audio_tokenizer.device = self.device
        self.audio_tokenizer.model.to(self.device)

        # NOTE: torch.compile disabled for BiCodec -- dynamic semantic sequence lengths
        # cause repeated recompilation (~120s each), worse than eager mode.
        # torch.compile works best with static shapes. If needed, pre-warm with
        # representative lengths or use mode="default" instead of "reduce-overhead".
        # The multi-engine approach (Tier 3) provides more impactful gains.

        print(f"✅ BiCodec decoder initialized (sample rate: {AUDIO_SAMPLE_RATE}Hz)")
        if enable_batching:
            print(f"   ℹ  Batching enabled (for streaming support)")

    def decode(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
    ) -> Optional[np.ndarray]:
        """
        Decode BiCodec tokens to audio waveform.

        CRITICAL: BiCodec expects EXACTLY 32 global tokens!
        The decoder pools them via speaker_encoder → d_vector, then broadcasts
        via d_vector.unsqueeze(-1) across the time dimension.

        Args:
            semantic_ids: List of semantic token IDs (variable length)
            global_ids: List of global token IDs (MUST be exactly 32!)

        Returns:
            Audio waveform as numpy array (float32, 16kHz mono)
            Shape: (samples,)
            Returns None if decode fails
        """
        if not semantic_ids or not global_ids:
            print(f"⚠  Empty token lists: semantic={len(semantic_ids)}, global={len(global_ids)}")
            return None

        # Validate global token count
        EXPECTED_GLOBAL_TOKENS = 32
        if len(global_ids) != EXPECTED_GLOBAL_TOKENS:
            print(f"❌ BiCodec requires EXACTLY {EXPECTED_GLOBAL_TOKENS} global tokens, got {len(global_ids)}")
            # Try to fix it
            if len(global_ids) > EXPECTED_GLOBAL_TOKENS:
                global_ids = global_ids[:EXPECTED_GLOBAL_TOKENS]
                print(f"   └─ Truncated to {EXPECTED_GLOBAL_TOKENS}")
            else:
                global_ids = global_ids + [0] * (EXPECTED_GLOBAL_TOKENS - len(global_ids))
                print(f"   └─ Padded to {EXPECTED_GLOBAL_TOKENS}")

        try:
            # Convert to tensors following official Spark-TTS implementation
            # global_token_ids shape: (1, 32) - ALWAYS 32!
            # pred_semantic_ids shape: (1, N) - variable length
            pred_semantic = torch.tensor(semantic_ids).long().unsqueeze(0).to(self.device)
            pred_global = torch.tensor(global_ids).long().unsqueeze(0).to(self.device)

            # Device migration done once in __init__, not per-call
            # Decode through BiCodec
            # BiCodecTokenizer.detokenize() expects:
            #   - global_tokens: (batch, global_dim) -> will unsqueeze(1) internally
            #   - semantic_tokens: (batch, latent_dim)
            with torch.inference_mode():
                wav_np = self.audio_tokenizer.detokenize(
                    pred_global,  # (1, seq) - correct shape!
                    pred_semantic  # (1, seq)
                )

            return wav_np

        except Exception as e:
            print(f"❌ BiCodec decode error: {e}")
            import traceback
            traceback.print_exc()
            return None

    def decode_to_bytes(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
    ) -> Optional[bytes]:
        """
        Decode BiCodec tokens to audio bytes (int16 PCM).

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs

        Returns:
            Audio as bytes (int16 PCM, 16kHz mono)
            Returns None if decode fails
        """
        audio = self.decode(semantic_ids, global_ids)

        if audio is None:
            return None

        # Clip and convert float32 to int16 PCM
        audio = np.clip(audio, -1.0, 1.0)
        audio_int16 = (audio * 32767).astype(np.int16)

        return audio_int16.tobytes()

    def validate_tokens(self, semantic_ids: List[int], global_ids: List[int]) -> bool:
        """
        Validate BiCodec tokens before decoding.

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs

        Returns:
            True if valid, False otherwise
        """
        # Check minimum length
        if not semantic_ids:
            print(f"❌ No semantic tokens")
            return False

        if not global_ids:
            print(f"❌ No global tokens")
            return False

        # Basic sanity checks
        if len(semantic_ids) < 1:
            print(f"❌ Too few semantic tokens: {len(semantic_ids)}")
            return False

        if len(global_ids) < 1:
            print(f"❌ Too few global tokens: {len(global_ids)}")
            return False

        return True

    def get_audio_duration(self, semantic_ids: List[int]) -> float:
        """
        Estimate audio duration from semantic tokens.

        Args:
            semantic_ids: List of semantic token IDs

        Returns:
            Estimated duration in seconds
        """
        # BiCodec generates audio based on semantic tokens
        # Approximate duration calculation (may need tuning)
        # This is an estimate and actual duration depends on model output
        estimated_samples = len(semantic_ids) * 320  # Rough estimate
        return estimated_samples / AUDIO_SAMPLE_RATE

    def decode_streaming(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
        use_sliding_window: bool = False,
        trim_warmup: bool = False,
    ) -> Optional[bytes]:
        """
        Decode BiCodec tokens with streaming support.

        CRITICAL: BiCodec decoder expects EXACTLY 32 global tokens always!
        The decoder internally broadcasts d_vector via d_vector.unsqueeze(-1).

        From sparktts/models/bicodec.py line 184-186:
            d_vector = self.speaker_encoder.detokenize(global_tokens)  # Expects 32 tokens
            x = self.prenet(z_q, d_vector)
            x = x + d_vector.unsqueeze(-1)  # Broadcasts across time automatically!

        Args:
            semantic_ids: List of semantic token IDs (variable length, 50 TPS)
            global_ids: List of global token IDs (MUST be exactly 32 tokens!)
            use_sliding_window: If True, return only middle samples
            trim_warmup: Legacy parameter (not used for BiCodec)

        Returns:
            Audio bytes (int16 PCM) or None if decode fails
        """
        # Minimum semantic tokens needed for stable decode
        MIN_SEMANTIC_TOKENS = 8

        if len(semantic_ids) < MIN_SEMANTIC_TOKENS:
            print(f"⚠  Too few semantic tokens ({len(semantic_ids)}) for decode, need >= {MIN_SEMANTIC_TOKENS}")
            return None

        # CRITICAL: Ensure we have exactly 32 global tokens
        EXPECTED_GLOBAL_TOKENS = 32
        if len(global_ids) != EXPECTED_GLOBAL_TOKENS:
            print(f"⚠  WARNING: Got {len(global_ids)} global tokens, expected {EXPECTED_GLOBAL_TOKENS}")
            if len(global_ids) > EXPECTED_GLOBAL_TOKENS:
                # Truncate to first 32
                global_ids = global_ids[:EXPECTED_GLOBAL_TOKENS]
                print(f"   └─ Truncated to {EXPECTED_GLOBAL_TOKENS} tokens")
            else:
                # Pad with zeros if we somehow have fewer
                global_ids = global_ids + [0] * (EXPECTED_GLOBAL_TOKENS - len(global_ids))
                print(f"   └─ Padded to {EXPECTED_GLOBAL_TOKENS} tokens")

        # Decode with EXACTLY 32 global tokens + variable semantic tokens
        # The decoder handles broadcasting internally!
        audio = self.decode(semantic_ids, global_ids)

        if audio is None:
            return None

        # If using sliding window, return only the middle portion
        if use_sliding_window and len(audio) > 4096:
            # Keep middle samples for overlap-add streaming
            # BiCodec produces ~320 samples per semantic token (16kHz / 50 TPS)
            total_samples = len(audio)
            keep_samples = min(4096, total_samples // 2)
            start = (total_samples - keep_samples) // 2
            end = start + keep_samples
            audio = audio[start:end]

        # Convert to bytes
        audio = np.clip(audio, -1.0, 1.0)
        audio_int16 = (audio * 32767).astype(np.int16)

        return audio_int16.tobytes()

    async def decode_single_async(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
        trim_warmup: bool = False,
        use_sliding_window: bool = False,
    ) -> Optional[bytes]:
        """
        Async wrapper for streaming decode - runs in executor to avoid blocking event loop.

        OPTIMIZATION: Previously ran synchronously, blocking all concurrent coroutines
        during GPU decode (~20-50ms). Now uses run_in_executor to unblock the event loop
        so other streams can progress while this decode runs on GPU.

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs
            trim_warmup: Legacy parameter (not used)
            use_sliding_window: Use sliding window mode

        Returns:
            Audio bytes (int16 PCM) or None
        """
        import asyncio
        loop = asyncio.get_event_loop()
        return await loop.run_in_executor(
            None,  # Default ThreadPoolExecutor
            self.decode_streaming,
            semantic_ids,
            global_ids,
            use_sliding_window,
            trim_warmup,
        )


sed -n '1,320p' veena3modal/core/token_utils.py


Output:
Chunk ID: eff1ad
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1578
Output:
"""
BiCodec Token Utilities - Incremental Parsing

Provides O(1) per-token parsing instead of O(n) decode + regex scanning.

The key optimization: instead of:
    tokenizer.decode(all_tokens)  # O(n)
    re.findall(entire_text)       # O(n)

We do:
    for new_token_id in new_tokens:  # Only new tokens
        parsed = token_cache.get(token_id)  # O(1) lookup

This eliminates the O(n²) CPU burn in the streaming hot loop.
"""

from typing import Dict, Optional, Tuple, List
import re

class BiCodecTokenParser:
    """
    Caching parser for BiCodec tokens.

    Converts vLLM token IDs to (type, value) tuples efficiently.

    BiCodec tokens in the vocabulary look like:
        - <|bicodec_semantic_123|> -> ("semantic", 123)
        - <|bicodec_global_7|>     -> ("global", 7)

    Usage:
        parser = BiCodecTokenParser(tokenizer)

        # In hot loop (O(1) per token):
        for token_id in new_token_ids:
            result = parser.parse(token_id)
            if result:
                token_type, value = result
                if token_type == "semantic":
                    semantic_buffer.append(value)
                elif token_type == "global":
                    global_buffer.append(value)
    """

    # Regex patterns for parsing token strings
    SEMANTIC_PATTERN = re.compile(r"^<\|bicodec_semantic_(\d+)\|>$")
    GLOBAL_PATTERN = re.compile(r"^<\|bicodec_global_(\d+)\|>$")

    def __init__(self, tokenizer):
        """
        Initialize parser with tokenizer.

        Args:
            tokenizer: HuggingFace tokenizer with BiCodec special tokens
        """
        self.tokenizer = tokenizer
        # Cache: token_id -> Optional[Tuple[str, int]]
        # None means "not a BiCodec token" (cached negative)
        self._cache: Dict[int, Optional[Tuple[str, int]]] = {}

        # Pre-warm cache with known BiCodec token ranges if possible
        # This is optional but can speed up first-pass parsing
        self._prewarm_cache()

    def _prewarm_cache(self) -> None:
        """
        Pre-populate cache with BiCodec tokens from vocabulary.

        This avoids lazy cache misses during streaming.
        """
        try:
            vocab = self.tokenizer.get_vocab()
            for token_str, token_id in vocab.items():
                # Check if it's a BiCodec token
                if token_str.startswith("<|bicodec_"):
                    parsed = self._parse_token_string(token_str)
                    if parsed:
                        self._cache[token_id] = parsed
        except Exception:
            # If vocab access fails, we'll rely on lazy caching
            pass

    def _parse_token_string(self, token_str: str) -> Optional[Tuple[str, int]]:
        """
        Parse a token string into (type, value) tuple.

        Args:
            token_str: Token string like "<|bicodec_semantic_123|>"

        Returns:
            ("semantic", 123) or ("global", 7) or None if not BiCodec token
        """
        # Try semantic pattern
        match = self.SEMANTIC_PATTERN.match(token_str)
        if match:
            return ("semantic", int(match.group(1)))

        # Try global pattern
        match = self.GLOBAL_PATTERN.match(token_str)
        if match:
            return ("global", int(match.group(1)))

        return None

    def parse(self, token_id: int) -> Optional[Tuple[str, int]]:
        """
        Parse a single token ID into (type, value) tuple.

        O(1) for cached tokens, O(1) amortized for uncached.

        Args:
            token_id: vLLM token ID

        Returns:
            ("semantic", value) or ("global", value) or None
        """
        # Check cache first
        if token_id in self._cache:
            return self._cache[token_id]

        # Cache miss - decode and parse
        try:
            token_str = self.tokenizer.decode([token_id], skip_special_tokens=False)
            parsed = self._parse_token_string(token_str.strip())
            self._cache[token_id] = parsed
            return parsed
        except Exception:
            # Mark as non-BiCodec to avoid repeated failures
            self._cache[token_id] = None
            return None

    def parse_incremental(
        self,
        new_token_ids: List[int],
        semantic_buffer: List[int],
        global_buffer: List[int],
    ) -> Tuple[List[int], List[int]]:
        """
        Parse new tokens and append to existing buffers.

        This is the main method for incremental streaming:
        - Only processes NEW tokens (not entire history)
        - Modifies buffers in-place for efficiency
        - Returns the updated buffers (same objects)

        Args:
            new_token_ids: Only the new token IDs since last call
            semantic_buffer: Existing semantic token values (modified in-place)
            global_buffer: Existing global token values (modified in-place)

        Returns:
            Tuple of (semantic_buffer, global_buffer) - same objects, modified
        """
        for token_id in new_token_ids:
            parsed = self.parse(token_id)
            if parsed:
                token_type, value = parsed
                if token_type == "semantic":
                    semantic_buffer.append(value)
                elif token_type == "global":
                    global_buffer.append(value)

        return semantic_buffer, global_buffer

    def get_cache_stats(self) -> Dict[str, int]:
        """
        Return cache statistics for debugging.

        Returns:
            Dict with cache size and breakdown
        """
        total = len(self._cache)
        bicodec_count = sum(1 for v in self._cache.values() if v is not None)
        return {
            "total_cached": total,
            "bicodec_tokens": bicodec_count,
            "non_bicodec_tokens": total - bicodec_count,
        }

def create_bicodec_parser(tokenizer) -> BiCodecTokenParser:
    """
    Factory function to create a BiCodec token parser.

    Args:
        tokenizer: HuggingFace tokenizer

    Returns:
        Configured BiCodecTokenParser instance
    """
    return BiCodecTokenParser(tokenizer)


sed -n '150,280p' veena3modal/tests/unit/test_logging.py


Output:
Chunk ID: 5d095a
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 757
Output:
        event = create_lifecycle_event(
            event_type="request_failed",
            request_id="req-005",
            status_code=500,
            error_code="GENERATION_FAILED",
            error_message="Model inference timeout",
        )

        assert event["event"] == "request_failed"
        assert event["error_code"] == "GENERATION_FAILED"

class TestJSONFormatter:
    """Test JSON log formatting."""

    def test_json_formatter_output(self):
        """Formatter should produce valid JSON."""
        from veena3modal.shared.logging import JSONFormatter

        formatter = JSONFormatter()

        # Create a log record
        record = logging.LogRecord(
            name="test",
            level=logging.INFO,
            pathname="test.py",
            lineno=10,
            msg="Test message",
            args=(),
            exc_info=None,
        )
        record.request_id = "req-json-001"
        record.custom_field = "value"

        output = formatter.format(record)

        # Should be valid JSON
        parsed = json.loads(output)
        assert parsed["message"] == "Test message"
        assert parsed["level"] == "INFO"
        assert parsed["request_id"] == "req-json-001"

    def test_json_formatter_handles_exceptions(self):
        """Formatter should include exception info."""
        from veena3modal.shared.logging import JSONFormatter

        formatter = JSONFormatter()

        try:
            raise ValueError("Test error")
        except ValueError:
            import sys
            exc_info = sys.exc_info()

        record = logging.LogRecord(
            name="test",
            level=logging.ERROR,
            pathname="test.py",
            lineno=10,
            msg="Error occurred",
            args=(),
            exc_info=exc_info,
        )

        output = formatter.format(record)
        parsed = json.loads(output)

        assert "exception" in parsed
        assert "ValueError" in parsed["exception"]

class TestRequestContext:
    """Test request context management."""

    def test_set_and_get_request_id(self):
        """Set and retrieve request_id from context."""
        from veena3modal.shared.logging import set_request_context, get_request_context, clear_request_context

        clear_request_context()

        set_request_context(request_id="ctx-001", user_id="user-123")

        ctx = get_request_context()
        assert ctx["request_id"] == "ctx-001"
        assert ctx["user_id"] == "user-123"

        clear_request_context()

    def test_clear_context(self):
        """Clear request context."""
        from veena3modal.shared.logging import set_request_context, get_request_context, clear_request_context

        set_request_context(request_id="ctx-002")
        clear_request_context()

        ctx = get_request_context()
        assert ctx.get("request_id") is None


Output:
Chunk ID: a8d68d
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 115
Output:
# Veena3 TTS Core Requirements
# PyTorch and ML dependencies
torch>=2.0.0
torchaudio>=2.0.0
transformers>=4.30.0
accelerate>=0.20.0

# vLLM for efficient inference
vllm>=0.6.0

# Audio processing
soundfile>=0.12.0
librosa>=0.10.0
scipy>=1.10.0
numpy>=1.24.0

# Hugging Face Hub for model downloads
huggingface-hub[cli]>=0.24.0  # Includes huggingface-cli command

# Tokenizers
tiktoken>=0.5.0
sentencepiece>=0.1.99

# Utilities
tqdm>=4.65.0
pydantic>=2.0.0


rg -n "generate_speech_indic\(|generate_speech\(" veena3modal | sed -n '1,220p'


Output:
Chunk ID: 295173
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 323
Output:
veena3modal/services/tts_runtime.py:347:async def generate_speech(
veena3modal/services/tts_runtime.py:397:        audio_bytes = await runtime.pipeline.generate_speech_indic(
veena3modal/services/tts_runtime.py:578:            audio_bytes = await runtime.pipeline.generate_speech_indic(
veena3modal/core/pipeline.py:55:    async def generate_speech(
veena3modal/core/pipeline.py:167:    async def generate_speech_indic(
veena3modal/core/pipeline.py:182:        It delegates to generate_speech() with the same implementation.
veena3modal/core/pipeline.py:202:        return await self.generate_speech(
veena3modal/api/fastapi_app.py:330:                audio_bytes, metrics = await tts_runtime.generate_speech(
veena3modal/processing/long_text_processor.py:222:            audio_bytes = await self.pipeline.generate_speech_indic(
veena3modal/tests/performance/test_benchmarks.py:67:        audio_bytes, metrics = await tts_runtime.generate_speech(
veena3modal/tests/performance/test_benchmarks.py:118:        audio_bytes, metrics = await tts_runtime.generate_speech(
veena3modal/tests/performance/test_benchmarks.py:182:            audio_bytes, metrics = await tts_runtime.generate_speech(
veena3modal/tests/performance/test_benchmarks.py:239:            await tts_runtime.generate_speech(


sed -n '150,280p' veena3modal/processing/long_text_processor.py


Output:
Chunk ID: 515e6c
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 1233
Output:
        seed: Optional[int] = None,
        sample_rate: int = 16000,  # BiCodec uses 16kHz, not 24kHz
    ) -> Optional[bytes]:
        """
        Generate audio for long text with chunking and stitching.

        Process:
        1. Chunk text at natural boundaries
        2. Generate audio for each chunk
        3. Stitch audio with crossfade

        Args:
            text: Long text to synthesize
            speaker: Speaker name (lipakshi, reet, etc.)
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max tokens per chunk
            seed: Random seed (same seed = same voice across chunks)
            sample_rate: Audio sample rate

        Returns:
            Stitched audio bytes (int16 PCM) or None if generation fails
        """
        # Detect language mix for logging
        lang_info = self.chunker.detect_language_mix(text)

        logger.info(
            f"Processing long text with chunking",
            extra={
                "text_length": len(text),
                "primary_language": lang_info['primary'],
                "lang_mix": lang_info,
                "speaker": speaker,
            }
        )

        # 1. Chunk text
        text_chunks = self.chunker.chunk_text(text)

        logger.info(
            f"Text chunked into {len(text_chunks)} chunks",
            extra={
                "num_chunks": len(text_chunks),
                "chunk_sizes": [len(chunk) for chunk in text_chunks],
            }
        )

        # 2. Generate audio for each chunk
        audio_chunks = []

        # Log summary at start
        logger.info(
            f"🎵 Generating {len(text_chunks)} chunks for {len(text)} chars",
            extra={
                "total_chunks": len(text_chunks),
                "total_chars": len(text),
                "speaker": speaker,
            }
        )

        for i, chunk in enumerate(text_chunks):
            # Calculate appropriate max_tokens for this chunk
            # BiCodec semantic tokens: ~50 tokens/second, ~5 words/second = ~10 tokens/word
            # Rough estimate: 5 chars/word, so ~2 tokens/char for audio generation
            # Add buffer for safety: multiply by 3 to handle variations
            chunk_max_tokens = min(
                int(len(chunk) * 3),  # ~2 tokens/char * 1.5 safety buffer = 3
                4096  # Cap at model's max context length
            )
            chunk_max_tokens = max(chunk_max_tokens, 100)  # Ensure minimum

            # Generate audio for this chunk (returns WAV format with header)
            audio_bytes = await self.pipeline.generate_speech_indic(
                speaker=speaker,
                text=chunk,
                temperature=temperature,
                top_k=top_k,  # Added for Spark TTS
                top_p=top_p,
                max_tokens=chunk_max_tokens,  # Use calculated limit, not global max_tokens
                repetition_penalty=repetition_penalty,  # Prevent token repetition
                seed=seed,  # Use same seed for consistent voice across chunks
            )

            if audio_bytes is None:
                logger.error(
                    f"❌ Failed to generate audio for chunk {i+1}/{len(text_chunks)}",
                    extra={
                        "chunk_index": i,
                        "chunk_length": len(chunk),
                        "chunk_text_preview": chunk[:150],
                        "total_chunks": len(text_chunks),
                    }
                )
                return None

            # Strip WAV header (44 bytes) to get raw PCM data for stitching
            pcm_data = audio_bytes[44:] if len(audio_bytes) > 44 else audio_bytes
            audio_chunks.append(pcm_data)

        # 3. Stitch audio with crossfade
        stitched_pcm = self.stitch_audio_chunks(
            audio_chunks,
            sample_rate=sample_rate,
            crossfade_ms=self.CROSSFADE_MS,
        )

        # 4. Add WAV header to stitched PCM data
        wav_header = create_wav_header(
            data_size=len(stitched_pcm),
            sample_rate=sample_rate,
            channels=1,
            bits_per_sample=16
        )
        final_audio = wav_header + stitched_pcm

        # Calculate total audio metrics
        total_audio_duration = len(stitched_pcm) / (sample_rate * 2)

        logger.info(
            f"✅ Generated {len(text_chunks)} chunks → {total_audio_duration:.1f}s audio",
            extra={
                "total_chunks": len(text_chunks),
                "audio_duration": total_audio_duration,
                "audio_size_mb": len(final_audio) / (1024 * 1024),
            }
        )

        return final_audio

    def stitch_audio_chunks(
        self,


rg -n "generate_speech_streaming\(" veena3modal | sed -n '1,120p'


Output:
Chunk ID: 817122
Wall time: 0.0505 seconds
Process exited with code 0
Original token count: 115
Output:
veena3modal/services/tts_runtime.py:625:async def generate_speech_streaming(
veena3modal/api/fastapi_app.py:550:                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
veena3modal/api/websocket_handler.py:342:            async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
veena3modal/tests/performance/test_benchmarks.py:87:        async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(


I’m going to add request-level `request_id` propagation into streaming internals so benchmark logs from server/client can be correlated per request, and then update both API and websocket callers. Next I’ll patch all necessary files in one change set.


sed -n '300,520p' veena3modal/api/websocket_handler.py


Output:
Chunk ID: 2a9563
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 1775
Output:
        chunks_sent = 0
        total_bytes = 0
        first_chunk_time = None
        final_metrics = {}

        # Start a task to listen for cancel messages
        async def listen_for_cancel():
            nonlocal cancelled
            try:
                while not cancelled:
                    try:
                        msg = await asyncio.wait_for(
                            websocket.receive_text(),
                            timeout=0.1
                        )
                        data = json.loads(msg)
                        if data.get("event") == "cancel":
                            cancelled = True
                            logger.info(f"Client requested cancel: {request_id}")
                            break
                        elif data.get("event") == "ping":
                            await websocket.send_text(json.dumps({"event": "pong"}))
                    except asyncio.TimeoutError:
                        continue
                    except Exception:
                        break
            except Exception:
                pass

        cancel_task = asyncio.create_task(listen_for_cancel())

        try:
            # Apply text normalization if enabled
            text = req.text
            if req.normalize:
                try:
                    from veena3modal.processing.text_normalizer import normalize_text
                    text = normalize_text(text)
                except ImportError:
                    pass  # Normalizer not available

            # Stream audio chunks
            async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
                text=text,
                speaker=speaker,
                temperature=req.temperature,
                top_k=req.top_k,
                top_p=req.top_p,
                max_tokens=req.max_tokens,
                repetition_penalty=req.repetition_penalty,
                seed=req.seed,
                enable_chunking=req.chunking,
            ):
                if cancelled:
                    logger.info(f"Generation cancelled: {request_id}")
                    break

                final_metrics = metrics
                chunks_sent += 1
                total_bytes += len(audio_chunk)

                # Record TTFB on first chunk
                if first_chunk_time is None:
                    first_chunk_time = time.time()
                    ttfb_ms = int((first_chunk_time - start_time) * 1000)
                    log_first_audio_emitted(
                        request_id=request_id,
                        ttfb_ms=ttfb_ms,
                        chunk_size_bytes=len(audio_chunk),
                    )
                    record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)

                # Send audio as binary frame
                await websocket.send_bytes(audio_chunk)

                # Send progress every 10 chunks
                if chunks_sent % 10 == 0:
                    await websocket.send_text(create_progress_message(
                        chunks_sent=chunks_sent,
                        bytes_sent=total_bytes,
                        elapsed_ms=int((time.time() - start_time) * 1000),
                    ))

        finally:
            cancel_task.cancel()
            try:
                await cancel_task
            except asyncio.CancelledError:
                pass

        # === SEND COMPLETION ===
        total_time = time.time() - start_time
        audio_duration = final_metrics.get("audio_duration_seconds", 0)
        rtf = total_time / audio_duration if audio_duration > 0 else 0

        completion_metrics = {
            "request_id": request_id,
            "chunks_sent": chunks_sent,
            "total_bytes": total_bytes,
            "audio_duration_seconds": round(audio_duration, 2),
            "total_time_seconds": round(total_time, 2),
            "ttfb_ms": int((first_chunk_time - start_time) * 1000) if first_chunk_time else 0,
            "rtf": round(rtf, 3),
            "cancelled": cancelled,
        }

        await websocket.send_text(create_complete_message(
            request_id=request_id,
            metrics=completion_metrics,
        ))

        # Log completion
        if not cancelled:
            log_request_completed(
                request_id=request_id,
                status_code=200,
                total_duration_ms=int(total_time * 1000),
                audio_duration_seconds=audio_duration,
                rtf=rtf,
                chunks_sent=chunks_sent,
            )
            metrics_request_completed(
                status_code=200,
                duration_seconds=total_time,
                speaker=speaker,
                stream=True,
            )
            record_rtf(rtf=rtf, speaker=speaker)
            record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
            record_chunks_sent(chunks=chunks_sent, speaker=speaker)

        # Close connection gracefully
        await websocket.close(code=1000)

    except Exception as e:
        logger.exception(f"WebSocket error: {e}")
        log_request_failed(
            request_id=request_id,
            status_code=500,
            error_code="WEBSOCKET_ERROR",
            error_message=str(e),
        )
        metrics_request_failed(
            status_code=500,
            error_code="WEBSOCKET_ERROR",
        )

        try:
            await websocket.send_text(create_error_message(
                code="INTERNAL_ERROR",
                message=str(e),
                request_id=request_id,
            ))
            await websocket.close(code=1011)
        except Exception:
            pass

    finally:
        clear_request_context()

def add_websocket_routes(app):
    """
    Add WebSocket routes to FastAPI app.

    Call this from create_app() to enable WebSocket support.

    Args:
        app: FastAPI application instance
    """
    from fastapi import WebSocket, Query

    @app.websocket("/v1/tts/ws")
    async def websocket_tts(
        websocket: WebSocket,
        api_key: Optional[str] = Query(None, alias="api_key"),
    ):
        """
        WebSocket endpoint for TTS streaming.

        Connect with optional api_key query parameter:
        ws://host/v1/tts/ws?api_key=your-key

        Then send JSON request:
        {"text": "Hello world", "speaker": "male1"}

        Receive:
        - JSON header message
        - Binary audio chunks
        - JSON progress messages (every 10 chunks)
        - JSON completion message
        """
        # Also check Authorization header
        auth_header = websocket.headers.get("authorization", "")
        if auth_header.startswith("Bearer "):
            api_key = auth_header[7:]

        await handle_websocket_tts(websocket, api_key)

    logger.info("WebSocket route /v1/tts/ws registered")


Input:
*** Begin Patch
*** Update File: veena3modal/core/pipeline.py
@@
 import asyncio
 import logging
 import re
+import os
+from statistics import median, mean
 from typing import Optional, List, Dict, Any, Tuple
 from vllm import SamplingParams
 import torch
@@
 logger = logging.getLogger(__name__)
+
+
+def _detailed_timing_enabled() -> bool:
+    """
+    Return True when benchmark logging is enabled.
+
+    Controlled by environment variable:
+      VEENA3_TIMING_LOGS=true|1
+    """
+    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
@@
     async def generate_speech(
         self,
         speaker: str,
         text: str,
         temperature: float = DEFAULT_TEMPERATURE,
         top_k: int = DEFAULT_TOP_K,
@@
         output sample rate.
+        return_timing: If True, returns (audio_bytes, metrics_dict) instead of
+            audio bytes only.
@@
         return:
             Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed
+            If return_timing is True, returns:
+                (audio_bytes, metrics_dict)
         """
+        emit_timing = return_timing or _detailed_timing_enabled()
+
         # Build prompt using Spark TTS format
         prompt = self.prompt_builder.build_prefix(speaker, text)
@@
         # Use vLLM engine to generate
         results_generator = self.model.engine.generate(
             prompt=prompt,
             sampling_params=sampling_params,
             request_id=request_id,
         )
-
-        # Collect all outputs
+
+        # Collect generator outputs and track per-batch token timings
         final_output = None
-        async for request_output in results_generator:
-            final_output = request_output
+        token_batches = []
+        batch_wall_ms = []
+        total_token_ids = 0
+        first_token_ms = None
+        t_start = time.time()
+        iterator = results_generator.__aiter__()
+        while True:
+            wall_start = time.time()
+            start_event = None
+            end_event = None
+            if torch.cuda.is_available():
+                torch.cuda.synchronize()
+                start_event = torch.cuda.Event(enable_timing=True)
+                end_event = torch.cuda.Event(enable_timing=True)
+                start_event.record()
+
+            try:
+                request_output = await iterator.__anext__()
+            except StopAsyncIteration:
+                break
+
+            wall_end = time.time()
+            batch_ms = (wall_end - wall_start) * 1000
+            batch_wall_ms.append(batch_ms)
+
+            if start_event is not None and end_event is not None:
+                end_event.record()
+                torch.cuda.synchronize()
+                batch_gpu_ms = start_event.elapsed_time(end_event)
+            else:
+                batch_gpu_ms = None
+
+            total_generated = len(request_output.outputs[0].token_ids)
+            new_tokens = max(0, total_generated - total_token_ids)
+            total_token_ids = total_generated
+            token_batches.append(new_tokens)
+
+            if first_token_ms is None:
+                first_token_ms = (wall_end - t_start) * 1000
+
+            if emit_timing:
+                logger.info(
+                    "tts_llm_batch",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "batch_index": len(token_batches),
+                        "batch_tokens": new_tokens,
+                        "llm_batch_wall_ms": round(batch_ms, 3),
+                        "llm_batch_gpu_ms": round(batch_gpu_ms, 3) if batch_gpu_ms is not None else None,
+                    },
+                )
+
+            final_output = request_output
+
+        if final_output is None:
+            if emit_timing:
+                logger.warning(
+                    "tts_llm_no_output",
+                    extra={"speaker": speaker, "request_id": request_id},
+                )
+            print(f"❌ Generation failed: no output")
+            return (None, {
+                "ttfb_ms": 0,
+                "token_total": 0,
+                "batch_count": 0,
+            }) if return_timing else None

         # Extract generated text (contains BiCodec tokens)
+        parse_start = time.time()
         generated_text = final_output.outputs[0].text
         generated_ids = final_output.outputs[0].token_ids
@@
         if not semantic_ids or not global_ids:
+            if emit_timing:
+                logger.error(
+                    "tts_bicodec_parse_failed",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "semantic_count": len(semantic_ids),
+                        "global_count": len(global_ids),
+                        "token_total": len(generated_ids),
+                    },
+                )
             logger.error(
                 "❌ No BiCodec tokens found in output",
                 extra={
                     "speaker": speaker,
                     "text_length": len(text),
@@
                 logger.error("❌ BiCodec token validation failed", extra={
                     "speaker": speaker,
                     "text_length": len(text),
                     "semantic_tokens": len(semantic_ids),
                     "global_tokens": len(global_ids),
                 })
                 return None

         # Decode to audio
+        t_decode_start = time.time()
         audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
+        t_decode_ms = (time.time() - t_decode_start) * 1000

         if audio_bytes is None:
+            if emit_timing:
+                logger.error(
+                    "tts_decode_failed",
+                    extra={"speaker": speaker, "request_id": request_id},
+                )
             logger.error("❌ BiCodec decode failed", extra={
                 "speaker": speaker,
                 "text_length": len(text),
             })
             return None
+
+        total_ms = (time.time() - t_start) * 1000
+        total_tokens = len(generated_ids)
+        parse_ms = (time.time() - parse_start) * 1000
+
+        timing_metrics = {
+            "request_id": request_id,
+            "speaker": speaker,
+            "text_length": len(text),
+            "batch_count": len(token_batches),
+            "token_total": total_tokens,
+            "token_batches": token_batches,
+            "ttfb_ms": int(first_token_ms or 0),
+            "llm_wall_ms": int(sum(batch_wall_ms)),
+            "llm_first_batch_ms": int(batch_wall_ms[0] * 1000) if batch_wall_ms else 0,
+            "llm_avg_batch_ms": int(mean(batch_wall_ms) if batch_wall_ms else 0),
+            "decode_cpu_ms": int(t_decode_ms),
+            "parse_ms": int(parse_ms),
+            "total_ms": int(total_ms),
+        }
+
+        if token_batches:
+            timing_metrics["tokens_per_batch_min"] = min(token_batches)
+            timing_metrics["tokens_per_batch_max"] = max(token_batches)
+            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))
+            if torch.cuda.is_available() and len(batch_wall_ms) > 0:
+                # Keep summary percentiles for wall-time only; GPU timings are already sampled above.
+                timing_metrics["llm_batch_count"] = len(batch_wall_ms)
+
+        if emit_timing:
+            logger.info(
+                "tts_llm_complete",
+                extra=timing_metrics,
+            )

         # Add WAV header
         from veena3modal.audio.utils import add_wav_header
         wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)
-
-        return wav_bytes
+
+        if return_timing:
+            return wav_bytes, timing_metrics
+        return wav_bytes
@@
         return:
             Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed
         """
         # Delegate to generate_speech() with the same implementation for Spark TTS
-        return await self.generate_speech(
+        return await self.generate_speech(
             speaker=speaker,
             text=text,
             temperature=temperature,
             top_k=top_k,
             top_p=top_p,
             max_tokens=max_tokens,
             repetition_penalty=repetition_penalty,
             seed=seed,
+            return_timing=return_timing,
         )
*** End Patch


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/veenaModal/veena3modal/core/pipeline.py:
        output sample rate.


Input:
*** Begin Patch
*** Delete File: veena3modal/core/pipeline.py
*** End Patch


Output: {"output":"Success. Updated the following files:\nD veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Add File: veena3modal/core/pipeline.py
+"""
+Spark TTS Generation Pipeline
+
+End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
+Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
+"""
+
+import asyncio
+import logging
+import os
+from statistics import mean, median
+from typing import Optional, List, Dict, Any, Tuple
+
+from vllm import SamplingParams
+import torch
+
+logger = logging.getLogger(__name__)
+
+from veena3modal.core.constants import (
+    TRAINING_STOP_TOKEN_IDS,
+    DEFAULT_TEMPERATURE,
+    DEFAULT_TOP_K,
+    DEFAULT_TOP_P,
+    DEFAULT_MAX_TOKENS,
+    DEFAULT_SEED,
+    AUDIO_SAMPLE_RATE,
+)
+
+
+def _detailed_timing_enabled() -> bool:
+    """
+    Return True when benchmark logs are enabled.
+
+    Controlled by environment variable:
+      VEENA3_TIMING_LOGS=true|1
+    """
+    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
+
+
+class SparkTTSPipeline:
+    """
+    End-to-end TTS pipeline for Spark TTS with BiCodec.
+
+    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
+    """
+
+    def __init__(
+        self,
+        model,
+        prompt_builder,
+        bicodec_decoder,
+    ):
+        """
+        Initialize pipeline.
+
+        Args:
+            model: SparkTTS Model instance
+            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
+            bicodec_decoder: BiCodecDecoder instance
+        """
+        self.model = model
+        self.prompt_builder = prompt_builder
+        self.bicodec_decoder = bicodec_decoder
+
+        print(f"🚀 SparkTTSPipeline initialized")
+
+    async def generate_speech(
+        self,
+        speaker: str,
+        text: str,
+        temperature: float = DEFAULT_TEMPERATURE,
+        top_k: int = DEFAULT_TOP_K,
+        top_p: float = DEFAULT_TOP_P,
+        max_tokens: int = DEFAULT_MAX_TOKENS,
+        repetition_penalty: float = 1.05,
+        seed: Optional[int] = None,
+        return_timing: bool = False,
+    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
+        """
+        Generate speech audio (non-streaming) using Spark TTS.
+
+        NOTE: This method signature changed from description-based to speaker-based
+        to align with Spark TTS architecture.
+
+        Args:
+            speaker: Speaker name (one of 12 predefined speakers)
+            text: Text to synthesize (with optional [emotion] tags)
+            temperature: Sampling temperature (default: 0.8 for Spark TTS)
+            top_k: Top-k sampling (default: 50)
+            top_p: Nucleus sampling (default: 1.0)
+            max_tokens: Max BiCodec tokens to generate (default: 2048)
+            seed: Random seed for reproducibility
+            return_timing: If true, returns (audio_bytes, metrics_dict)
+
+        Returns:
+            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
+            If return_timing is true, returns tuple(audio_bytes, metrics).
+        """
+        # Build prompt using Spark TTS format
+        prompt = self.prompt_builder.build_prefix(speaker, text)
+
+        # Configure sampling for Spark TTS
+        sampling_params = SamplingParams(
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            max_tokens=max_tokens,
+            repetition_penalty=repetition_penalty,
+            stop=TRAINING_STOP_TOKEN_IDS,
+            skip_special_tokens=False,
+            seed=seed,
+        )
+
+        # Use vLLM engine to generate
+        import uuid
+        request_id = f"req-{uuid.uuid4().hex[:12]}"
+
+        emit_timing = return_timing or _detailed_timing_enabled()
+        results_generator = self.model.engine.generate(
+            prompt=prompt,
+            sampling_params=sampling_params,
+            request_id=request_id,
+        )
+
+        # Collect all outputs and track generation timing
+        final_output = None
+        token_batches: List[int] = []
+        batch_wall_ms: List[float] = []
+        total_token_ids = 0
+        first_token_ms: Optional[float] = None
+        t_start = asyncio.get_event_loop().time()
+
+        iterator = results_generator.__aiter__()
+        while True:
+            wall_start = asyncio.get_event_loop().time()
+            start_event = None
+            end_event = None
+            if torch.cuda.is_available():
+                torch.cuda.synchronize()
+                start_event = torch.cuda.Event(enable_timing=True)
+                end_event = torch.cuda.Event(enable_timing=True)
+                start_event.record()
+
+            try:
+                request_output = await iterator.__anext__()
+            except StopAsyncIteration:
+                break
+
+            wall_end = asyncio.get_event_loop().time()
+            batch_ms = (wall_end - wall_start) * 1000
+            batch_wall_ms.append(batch_ms)
+
+            if start_event is not None and end_event is not None:
+                end_event.record()
+                torch.cuda.synchronize()
+                batch_gpu_ms = start_event.elapsed_time(end_event)
+            else:
+                batch_gpu_ms = None
+
+            total_generated = len(request_output.outputs[0].token_ids)
+            new_tokens = max(0, total_generated - total_token_ids)
+            total_token_ids = total_generated
+            token_batches.append(new_tokens)
+
+            if first_token_ms is None:
+                first_token_ms = (wall_end - t_start) * 1000
+
+            if emit_timing:
+                logger.info(
+                    "tts_llm_batch",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "batch_index": len(token_batches),
+                        "batch_tokens": new_tokens,
+                        "llm_batch_wall_ms": round(batch_ms, 3),
+                        "llm_batch_gpu_ms": round(batch_gpu_ms, 3) if batch_gpu_ms is not None else None,
+                    },
+                )
+            final_output = request_output
+
+        if final_output is None:
+            if emit_timing:
+                logger.warning(
+                    "tts_llm_no_output",
+                    extra={"speaker": speaker, "request_id": request_id},
+                )
+            print(f"❌ Generation failed: no output")
+            if return_timing:
+                return None, {"ttfb_ms": 0, "token_total": 0, "batch_count": 0}
+            return None
+
+        # Extract generated text (contains BiCodec tokens)
+        parse_start = asyncio.get_event_loop().time()
+        generated_text = final_output.outputs[0].text
+        generated_ids = final_output.outputs[0].token_ids
+
+        # Extract BiCodec tokens using regex
+        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)
+
+        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000
+
+        if not semantic_ids or not global_ids:
+            if emit_timing:
+                logger.error(
+                    "tts_bicodec_parse_failed",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "semantic_count": len(semantic_ids),
+                        "global_count": len(global_ids),
+                        "token_total": len(generated_ids),
+                    },
+                )
+            logger.error(
+                "❌ No BiCodec tokens found in output",
+                extra={
+                    "speaker": speaker,
+                    "text_length": len(text),
+                    "text_preview": text[:100],
+                    "generated_tokens": len(generated_ids),
+                    "semantic_tokens": len(semantic_ids),
+                    "global_tokens": len(global_ids),
+                    "generated_text_preview": generated_text[:500],
+                }
+            )
+            if return_timing:
+                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
+            return None
+
+        # Validate tokens
+        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
+            logger.error("❌ BiCodec token validation failed", extra={
+                "speaker": speaker,
+                "text_length": len(text),
+                "semantic_tokens": len(semantic_ids),
+                "global_tokens": len(global_ids),
+            })
+            if return_timing:
+                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
+            return None
+
+        # Decode to audio
+        decode_start = asyncio.get_event_loop().time()
+        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
+        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000
+
+        if audio_bytes is None:
+            if emit_timing:
+                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
+            logger.error("❌ BiCodec decode failed", extra={
+                "speaker": speaker,
+                "text_length": len(text),
+            })
+            if return_timing:
+                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
+            return None
+
+        total_ms = (asyncio.get_event_loop().time() - t_start) * 1000
+
+        timing_metrics: Dict[str, Any] = {
+            "request_id": request_id,
+            "speaker": speaker,
+            "text_length": len(text),
+            "batch_count": len(token_batches),
+            "token_total": len(generated_ids),
+            "token_batches": token_batches,
+            "ttfb_ms": int(first_token_ms or 0),
+            "llm_wall_ms": int(sum(batch_wall_ms)),
+            "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
+            "decode_cpu_ms": int(decode_ms),
+            "parse_ms": int(parse_ms),
+            "total_ms": int(total_ms),
+        }
+        if token_batches:
+            timing_metrics["tokens_per_batch_min"] = min(token_batches)
+            timing_metrics["tokens_per_batch_max"] = max(token_batches)
+            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))
+
+        if emit_timing:
+            logger.info("tts_llm_complete", extra=timing_metrics)
+
+        from veena3modal.audio.utils import add_wav_header
+        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)
+
+        if return_timing:
+            return wav_bytes, timing_metrics
+        return wav_bytes
+
+    async def generate_speech_indic(
+        self,
+        speaker: str,
+        text: str,
+        temperature: float = DEFAULT_TEMPERATURE,
+        top_k: int = DEFAULT_TOP_K,
+        top_p: float = DEFAULT_TOP_P,
+        max_tokens: int = DEFAULT_MAX_TOKENS,
+        repetition_penalty: float = 1.05,
+        seed: Optional[int] = None,
+        return_timing: bool = False,
+    ):
+        """
+        Generate speech audio for Spark TTS (speaker-based).
+
+        This method is kept for backward compatibility with existing API.
+        It delegates to generate_speech() with the same implementation.
+
+        Args:
+            speaker: Speaker name (one of 12 predefined speakers)
+            text: Text to synthesize with inline emotion tags
+                Examples:
+                - "Hello! Welcome."
+                - "Hello <laugh> this is fun!" (will be normalized)
+                - "नमस्ते! [excited] आज का दिन बहुत अच्छा है।"
+            temperature: Sampling temperature
+            top_k: Top-k sampling
+            top_p: Nucleus sampling
+            max_tokens: Max BiCodec tokens to generate
+            seed: Random seed for reproducibility
+            return_timing: If true, returns (audio_bytes, metrics_dict)
+
+        Returns:
+            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
+            If return_timing is true, returns tuple(audio_bytes, metrics).
+        """
+        # Delegate to generate_speech() with the same implementation for Spark TTS
+        return await self.generate_speech(
+            speaker=speaker,
+            text=text,
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            max_tokens=max_tokens,
+            repetition_penalty=repetition_penalty,
+            seed=seed,
+            return_timing=return_timing,
+        )
+
+    def _extract_bicodec_tokens(self, generated_text: str) -> Tuple[List[int], List[int]]:
+        """
+        Extract BiCodec semantic and global tokens from generated text using regex.
+
+        Spark TTS generates tokens in the format:
+        - <|bicodec_semantic_{id}|>
+        - <|bicodec_global_{id}|>
+
+        Args:
+            generated_text: Generated text containing BiCodec token markers
+
+        Returns:
+            Tuple of (semantic_ids, global_ids)
+            Returns ([], []) if no tokens found
+        """
+        # Extract semantic tokens
+        semantic_matches = re.findall(r"<\\|bicodec_semantic_(\\d+)\\|>", generated_text)
+        semantic_ids = [int(t) for t in semantic_matches]
+
+        # Extract global tokens
+        global_matches = re.findall(r"<\\|bicodec_global_(\\d+)\\|>", generated_text)
+        global_ids = [int(t) for t in global_matches]
+
+        # Only log if there's an issue
+        if not semantic_ids and not global_ids:
+            logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")
+
+        return semantic_ids, global_ids
*** End Patch


Output: {"output":"Success. Updated the following files:\nA veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


rg -n "r\"<\\\\\\|" veena3modal/core/pipeline.py


Output:
Chunk ID: 37ba1c
Wall time: 0.0512 seconds
Process exited with code 1
Original token count: 0
Output:


grep -n "bicodec_semantic" -n veena3modal/core/pipeline.py


Output:
Chunk ID: 6c3fc1
Wall time: 0.0538 seconds
Process exited with code 0
Original token count: 34
Output:
344:        - <|bicodec_semantic_{id}|>
355:        semantic_matches = re.findall(r"<\\|bicodec_semantic_(\\d+)\\|>", generated_text)


sed -n '344,365p' veena3modal/core/pipeline.py


Output:
Chunk ID: 835639
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 213
Output:
        - <|bicodec_semantic_{id}|>
        - <|bicodec_global_{id}|>

        Args:
            generated_text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
            Returns ([], []) if no tokens found
        """
        # Extract semantic tokens
        semantic_matches = re.findall(r"<\\|bicodec_semantic_(\\d+)\\|>", generated_text)
        semantic_ids = [int(t) for t in semantic_matches]

        # Extract global tokens
        global_matches = re.findall(r"<\\|bicodec_global_(\\d+)\\|>", generated_text)
        global_ids = [int(t) for t in global_matches]

        # Only log if there's an issue
        if not semantic_ids and not global_ids:
            logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")


Input:
*** Begin Patch
*** Update File: veena3modal/core/pipeline.py
@@
-        semantic_matches = re.findall(r"<\\|bicodec_semantic_(\\d+)\\|>", generated_text)
+        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", generated_text)
         semantic_ids = [int(t) for t in semantic_matches]

         # Extract global tokens
-        global_matches = re.findall(r"<\\|bicodec_global_(\\d+)\\|>", generated_text)
+        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
         global_ids = [int(t) for t in global_matches]
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
 import os
 import sys
 import time
 import logging
+from threading import Lock
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
@@
 logger = logging.getLogger(__name__)
+
+_PROFILE_REQUEST_TIMING = "VEENA3_TIMING_LOGS"
+_inflight_counter = 0
+_inflight_lock = Lock()
+
+
+def _timing_enabled() -> bool:
+    """Return True when detailed request timing should be emitted."""
+    return os.getenv(_PROFILE_REQUEST_TIMING, "").lower() in {"1", "true", "yes", "on"}
+
+
+def _with_inflight_delta(delta: int) -> int:
+    """Update in-flight request counter and return the new value."""
+    global _inflight_counter
+    with _inflight_lock:
+        _inflight_counter = max(0, _inflight_counter + delta)
+        return _inflight_counter
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
 async def generate_speech(
     text: str,
     speaker: str,
     temperature: float = 0.8,
@@
 ) -> Tuple[Optional[bytes], Dict[str, Any]]:
@@
-    start_time = time.time()
-
-    try:
-        # Generate audio using pipeline (always at 16kHz)
-        audio_bytes = await runtime.pipeline.generate_speech_indic(
-            speaker=speaker,
-            text=text,
-            temperature=temperature,
-            top_k=top_k,
-            top_p=top_p,
-            max_tokens=max_tokens,
-            repetition_penalty=repetition_penalty,
-            seed=seed,
-        )
-
-        generation_time = time.time() - start_time
-        metrics["generation_ms"] = int(generation_time * 1000)
-        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time
-
-        if audio_bytes:
+    active_requests = _with_inflight_delta(1)
+    start_time = time.time()
+
+    pipeline_timings = {}
+
+    try:
+        logger.info(
+            "tts_runtime_nonstream_start",
+            extra={
+                "speaker": speaker,
+                "text_length": len(text),
+                "request_inflight": active_requests,
+                "temperature": temperature,
+                "top_k": top_k,
+                "top_p": top_p,
+                "max_tokens": max_tokens,
+            },
+        )
+
+        gen_kwargs = dict(
+            speaker=speaker,
+            text=text,
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            max_tokens=max_tokens,
+            repetition_penalty=repetition_penalty,
+            seed=seed,
+        )
+
+        if _timing_enabled():
+            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
+                **gen_kwargs,
+                return_timing=True,
+            )
+        else:
+            audio_bytes = await runtime.pipeline.generate_speech_indic(**gen_kwargs)
+
+        generation_time = time.time() - start_time
+        metrics["generation_ms"] = int(generation_time * 1000)
+        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time
+
+        if pipeline_timings:
+            metrics.update({
+                "llm_token_total": pipeline_timings.get("token_total", 0),
+                "llm_batch_count": pipeline_timings.get("batch_count", 0),
+                "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
+                "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
+                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
+                "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
+                "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
+                "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
+                "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
+            })
+
+        if audio_bytes:
             # Apply super-resolution if requested and available
             sample_rate = 16000
             logger.info(f"SR check: output_sample_rate={output_sample_rate!r}, "
                        f"sr_service={runtime.sr_service is not None}, "
                        f"is_loaded={runtime.sr_service.is_loaded if runtime.sr_service else False}")
@@
                 audio_duration = (len(audio_bytes) - 44) / (sample_rate * 2)  # -44 for WAV header
                 metrics["audio_duration_seconds"] = max(0.0, audio_duration)
-
-        return audio_bytes, metrics
+
+        if _timing_enabled():
+            logger.info("tts_runtime_nonstream_complete", extra=metrics)
+
+        return audio_bytes, metrics

     except Exception as e:
         logger.error(f"Speech generation failed: {e}")
         raise
+    finally:
+        active_requests = _with_inflight_delta(-1)
+        if _timing_enabled():
+            logger.info(
+                "tts_runtime_nonstream_inflight",
+                extra={"active_requests": active_requests},
+            )
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
 ) -> Tuple[Optional[bytes], Dict[str, Any]]:
@@
-    start_time = time.time()
-
-    try:
-        # Import long text processor
+    active_requests = _with_inflight_delta(1)
+    start_time = time.time()
+
+    try:
+        if _timing_enabled():
+            logger.info(
+                "tts_runtime_chunking_start",
+                extra={
+                    "speaker": speaker,
+                    "text_length": len(text),
+                    "request_inflight": active_requests,
+                    "chunking": True,
+                },
+            )
+        # Import long text processor
         from veena3modal.processing.long_text_processor import LongTextProcessor
@@
-        return audio_bytes, metrics
+        if _timing_enabled():
+            metrics["active_requests"] = active_requests - 1
+            logger.info("tts_runtime_chunking_complete", extra=metrics)
+        return audio_bytes, metrics

     except Exception as e:
         logger.error(f"Chunked speech generation failed: {e}")
         raise
+    finally:
+        _with_inflight_delta(-1)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
 async def generate_speech_streaming(
     text: str,
     speaker: str,
     temperature: float = 0.8,
@@
     repetition_penalty: float = 1.05,
     seed: Optional[int] = None,
     enable_chunking: bool = True,
+    request_id: Optional[str] = None,
 ) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
     start_time = time.time()
     first_chunk_time = None
     total_pcm_bytes = 0
     sample_rate = 16000  # BiCodec sample rate
+    pipeline_metrics: Dict[str, Any] = {
+        "request_id": request_id,
+    }
+    active_requests = _with_inflight_delta(1)
+    if _timing_enabled():
+        logger.info(
+            "tts_runtime_stream_start",
+            extra={
+                "speaker": speaker,
+                "text_length": len(text),
+                "request_inflight": active_requests,
+                "chunking": enable_chunking,
+                "request_id": request_id,
+            },
+        )

     # Check if we need text chunking
@@
     if needs_chunking:
         # Chunked streaming with voice consistency (global token caching)
         metrics["text_chunked"] = True
+        if _timing_enabled():
+            pipeline_metrics["chunked"] = True
         async for audio_chunk, chunk_metrics in _stream_chunked_text(
             runtime, long_processor, text, speaker, temperature, top_k, top_p,
             max_tokens, repetition_penalty, seed, sample_rate
+            ,
         ):
             # First chunk: prepend WAV header
             if first_chunk_time is None:
                 first_chunk_time = time.time()
                 metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
@@
             metrics["chunks_sent"] += 1
             metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
             metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
+            if chunk_metrics:
+                pipeline_metrics.update(chunk_metrics)

             yield audio_chunk, metrics
     else:
         # Simple streaming (no text chunking)
@@
                 async for audio_chunk in stream_gen:
             # First chunk: prepend WAV header
             if first_chunk_time is None:
                 first_chunk_time = time.time()
                 metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
@@
             metrics["chunks_sent"] += 1
             metrics["total_bytes"] = total_pcm_bytes + 44
             metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
+            if _timing_enabled():
+                pipeline_metrics["active_requests_at_emit"] = active_requests

             yield audio_chunk, metrics

 async def _stream_chunked_text(
@@
     sample_rate: int,
 ) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


sed -n '640,840p' veena3modal/services/tts_runtime.py