Output:
Chunk ID: 120da5
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 2463
Output:
155:        "request_id",
297:        return Response(
313:        request_id = str(uuid.uuid4())
328:                        request_id=request_id,
330:                    headers={"X-Request-ID": request_id},
345:                        request_id=request_id,
347:                    headers={"X-Request-ID": request_id},
357:                    headers["X-Request-ID"] = request_id
363:                            request_id=request_id,
385:                        "request_id": request_id,
388:                headers={"X-Request-ID": request_id},
399:                        "request_id": request_id,
402:                headers={"X-Request-ID": request_id},
420:        set_request_context(request_id=request_id)
424:            request_id=request_id,
434:            return await _handle_streaming_request(req, request_id, text, speaker, start_time)
457:                    request_id=request_id,
470:                    request_id=request_id,
480:                            "request_id": request_id,
483:                    headers={"X-Request-ID": request_id},
499:                "X-Request-ID": request_id,
519:                request_id=request_id,
538:                request_id=request_id,
598:            return Response(
609:                request_id=request_id,
627:                        "request_id": request_id,
630:                headers={"X-Request-ID": request_id},
633:    async def _handle_streaming_request(req, request_id: str, text: str, speaker: str, start_time: float):
653:                        "request_id": request_id,
656:                headers={"X-Request-ID": request_id},
667:                        "request_id": request_id,
670:                headers={"X-Request-ID": request_id},
674:        set_request_context(request_id=request_id)
693:                    request_id=request_id,
705:                            request_id=request_id,
713:                            request_id=request_id,
737:                    request_id=request_id,
759:                    request_id=request_id,
780:            "X-Request-ID": request_id,

            return JSONResponse(
                status_code=500,
                content={
                    "error": {
                        "code": "INTERNAL_ERROR",
                        "message": str(e),
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

    async def _handle_streaming_request(req, request_id: str, text: str, speaker: str, start_time: float):
        """
        Handle streaming TTS request.

        Returns a StreamingResponse that yields WAV header + PCM chunks.
        True streaming: first bytes sent ASAP before full audio is generated.
        """
        from fastapi.responses import StreamingResponse
        from veena3modal.api.schemas import AudioFormat
        from veena3modal.services import tts_runtime

        # Check if streaming pipeline is available
        runtime = tts_runtime.get_runtime()
        if runtime is None or runtime.streaming_pipeline is None:
            return JSONResponse(
                status_code=503,
                content={
                    "error": {
                        "code": "STREAMING_UNAVAILABLE",
                        "message": "Streaming pipeline not initialized.",
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

        # Non-WAV formats not implemented yet (M5)
        if req.format != AudioFormat.WAV.value:
            return JSONResponse(
                status_code=501,
                content={
                    "error": {
                        "code": "FORMAT_NOT_IMPLEMENTED",
                        "message": f"Streaming format '{req.format}' not yet implemented. Use format=wav.",
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

        # Set request context for structured logging
        set_request_context(request_id=request_id)

        # Create async generator for streaming response
        async def audio_stream_generator():
            """Yields audio chunks as they're generated."""
            final_metrics = {}
            first_chunk_sent = False
            chunks_count = 0
            try:
                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
                    text=text,
                    speaker=speaker,
                    temperature=req.temperature,
                    top_k=req.top_k,
                    top_p=req.top_p,
                    max_tokens=req.max_tokens,
                    repetition_penalty=req.repetition_penalty,
                    seed=req.seed,
                    enable_chunking=req.chunking,
                    request_id=request_id,
                ):
                    final_metrics = metrics
                    chunks_count += 1

                    # Fire-and-forget sentence storage after first chunk (don't block TTFB)
                    if not first_chunk_sent:
                        first_chunk_sent = True
                        ttfb_ms = metrics.get("ttfb_ms", 0)

                        # Log first audio emitted (TTFB marker)
                        log_first_audio_emitted(
                            request_id=request_id,
                            ttfb_ms=ttfb_ms,
                            chunk_size_bytes=len(audio_chunk),
                        )
                        record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)

                        sentence_store = _get_sentence_store()
                        sentence_store.store_fire_and_forget(
                            request_id=request_id,
                            text=text,
                            speaker=speaker,
                            stream=True,
                            format=req.format,
                            temperature=req.temperature,
                            top_k=req.top_k,
                            top_p=req.top_p,
                            max_tokens=req.max_tokens,
                            repetition_penalty=req.repetition_penalty,
                            seed=req.seed,
                            text_chunked=metrics.get("text_chunked", False),
                            ttfb_ms=ttfb_ms,
                            # audio_duration_seconds not known yet for streaming
                        )

                    yield audio_chunk

                # Stream completed - log metrics
                total_time = time.time() - start_time
                audio_duration = final_metrics.get("audio_duration_seconds", 0)
                rtf = total_time / audio_duration if audio_duration > 0 else 0

                log_request_completed(
                    request_id=request_id,
                    status_code=200,
                    total_duration_ms=int(total_time * 1000),
                    audio_duration_seconds=audio_duration,
                    rtf=rtf,
                    chunks_sent=chunks_count,
                    **final_metrics,
                )
                metrics_request_completed(
                    status_code=200,
                    duration_seconds=total_time,
                    speaker=speaker,
                    stream=True,
                )
                record_rtf(rtf=rtf, speaker=speaker)
                record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
                record_chunks_sent(chunks=chunks_count, speaker=speaker)
                clear_request_context()
            except Exception as e:
                logger.exception(f"Streaming error: {e}")
                # Log streaming failure
                log_request_failed(
                    request_id=request_id,
                    status_code=500,
                    error_code="STREAMING_ERROR",
                    error_message=str(e),
                )
                metrics_request_failed(
                    status_code=500,
                    error_code="STREAMING_ERROR",
                    speaker=speaker,
                )
                clear_request_context()
                # Can't return error once streaming started
                # Best we can do is stop yielding
                return

        # Build response headers
        # Note: Some headers can't be set dynamically during streaming
        # TTFB and RTF will be logged but not in headers for streaming
        model_version = runtime.model_version if runtime else get_model_version()

        headers = {
            "X-Request-ID": request_id,
            "X-Model-Version": model_version,
            "X-Format": "wav",
            "X-Sample-Rate": str(req.sample_rate),
            "X-Stream": "true",
            "X-Chunking-Enabled": str(req.chunking).lower(),
            # Transfer-Encoding: chunked is set automatically by StreamingResponse
        }

        if req.seed is not None:
            headers["X-Seed"] = str(req.seed)

        return StreamingResponse(
            audio_stream_generator(),
            media_type="audio/wav",
            headers=headers,
        )

    # === WebSocket Support (M5b) ===
    try:
        from veena3modal.api.websocket_handler import add_websocket_routes
        add_websocket_routes(app)
        logger.info("WebSocket support enabled at /v1/tts/ws")
    except ImportError as e:
        logger.warning(f"WebSocket support not available: {e}")

    return app


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '1,320p' veena3modal/shared/logging.py


Output:
Chunk ID: 25a7b3
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1813
Output:
"""
Structured logging for Veena3 TTS service.

Features:
- JSON-formatted logs for structured queries
- Request ID tracking via context
- Lifecycle event helpers (request_received, first_audio_emitted, etc.)
- NO PII in logs (text content stored in Supabase, not logs)

Usage:
    from veena3modal.shared.logging import get_logger, log_event, set_request_context

    logger = get_logger(__name__)
    set_request_context(request_id="req-123")
    log_event(logger, "request_received", text_length=100, speaker="lipakshi")
"""

import logging
import json
import sys
import traceback
from datetime import datetime, timezone
from typing import Any, Dict, Optional
from contextvars import ContextVar

# Context variable for request-scoped data
_request_context: ContextVar[Dict[str, Any]] = ContextVar('request_context', default={})

def set_request_context(request_id: Optional[str] = None, **kwargs) -> None:
    """
    Set request context for current async task.

    Args:
        request_id: Unique request identifier
        **kwargs: Additional context fields (user_id, etc.)
    """
    ctx = {"request_id": request_id, **kwargs}
    _request_context.set(ctx)

def get_request_context() -> Dict[str, Any]:
    """Get current request context."""
    return _request_context.get()

def clear_request_context() -> None:
    """Clear request context after request completes."""
    _request_context.set({})

class JSONFormatter(logging.Formatter):
    """
    JSON log formatter for structured logging.

    Output format:
    {
        "timestamp": "2024-01-01T12:00:00.000Z",
        "level": "INFO",
        "logger": "veena3modal.api",
        "message": "Request received",
        "request_id": "req-123",
        ...extra fields...
    }
    """

    # Fields to exclude from extra (already handled separately)
    RESERVED_ATTRS = {
        'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
        'funcName', 'levelname', 'levelno', 'lineno', 'module', 'msecs',
        'message', 'msg', 'name', 'pathname', 'process', 'processName',
        'relativeCreated', 'stack_info', 'thread', 'threadName',
        'taskName',  # Python 3.12+
    }

    def format(self, record: logging.LogRecord) -> str:
        """Format log record as JSON."""
        # Build base log entry
        log_entry = {
            "timestamp": datetime.now(timezone.utc).isoformat(),
            "level": record.levelname,
            "logger": record.name,
            "message": record.getMessage(),
        }

        # Add request context if available
        ctx = get_request_context()
        if ctx.get("request_id"):
            log_entry["request_id"] = ctx["request_id"]

        # Add extra fields from record
        for key, value in record.__dict__.items():
            if key not in self.RESERVED_ATTRS and not key.startswith('_'):
                try:
                    # Ensure value is JSON serializable
                    json.dumps(value)
                    log_entry[key] = value
                except (TypeError, ValueError):
                    log_entry[key] = str(value)

        # Add exception info if present
        if record.exc_info:
            log_entry["exception"] = ''.join(traceback.format_exception(*record.exc_info))

        return json.dumps(log_entry, default=str)

def get_logger(name: str) -> logging.Logger:
    """
    Get a logger configured for structured JSON output.

    Args:
        name: Logger name (typically __name__)

    Returns:
        Configured logger instance
    """
    logger = logging.getLogger(name)

    # Only configure if no handlers exist (avoid duplicate setup)
    if not logger.handlers:
        handler = logging.StreamHandler(sys.stdout)
        handler.setFormatter(JSONFormatter())
        logger.addHandler(handler)
        logger.setLevel(logging.INFO)

    return logger

def log_event(
    logger: logging.Logger,
    event_type: str,
    level: int = logging.INFO,
    request_id: Optional[str] = None,
    **kwargs
) -> None:
    """
    Log a structured event.

    Args:
        logger: Logger instance
        event_type: Event name (e.g., "request_received", "first_audio_emitted")
        level: Log level (default: INFO)
        request_id: Override request_id from context
        **kwargs: Event-specific fields
    """
    extra = {"event": event_type, **kwargs}

    if request_id:
        extra["request_id"] = request_id

    logger.log(level, event_type, extra=extra)

def create_lifecycle_event(
    event_type: str,
    request_id: str,
    **kwargs
) -> Dict[str, Any]:
    """
    Create a lifecycle event dictionary.

    Lifecycle events:
    - request_received: TTS request started
    - auth_validated: API key validated
    - generation_started: Model inference began
    - first_audio_emitted: First audio chunk sent (TTFB)
    - request_completed: Request finished successfully
    - request_failed: Request failed with error

    Args:
        event_type: Event name
        request_id: Request identifier
        **kwargs: Event-specific fields

    Returns:
        Event dictionary ready for logging
    """
    return {
        "event": event_type,
        "request_id": request_id,
        "timestamp": datetime.now(timezone.utc).isoformat(),
        **kwargs
    }

# Pre-configured logger for the TTS service
tts_logger = get_logger("veena3modal.tts")

def log_request_received(
    request_id: str,
    text_length: int,
    speaker: str,
    stream: bool,
    format: str,
    **kwargs
) -> None:
    """Log request_received lifecycle event."""
    log_event(
        tts_logger,
        "request_received",
        request_id=request_id,
        text_length=text_length,
        speaker=speaker,
        stream=stream,
        format=format,
        **kwargs
    )

def log_first_audio_emitted(
    request_id: str,
    ttfb_ms: int,
    chunk_size_bytes: int,
    **kwargs
) -> None:
    """Log first_audio_emitted lifecycle event (TTFB marker)."""
    log_event(
        tts_logger,
        "first_audio_emitted",
        request_id=request_id,
        ttfb_ms=ttfb_ms,
        chunk_size_bytes=chunk_size_bytes,
        **kwargs
    )

def log_request_completed(
    request_id: str,
    status_code: int,
    total_duration_ms: int,
    audio_duration_seconds: float,
    rtf: float,
    chunks_sent: int = 1,
    **kwargs
) -> None:
    """Log request_completed lifecycle event."""
    log_event(
        tts_logger,
        "request_completed",
        request_id=request_id,
        status_code=status_code,
        total_duration_ms=total_duration_ms,
        audio_duration_seconds=audio_duration_seconds,
        rtf=rtf,
        chunks_sent=chunks_sent,
        **kwargs
    )

def log_request_failed(
    request_id: str,
    status_code: int,
    error_code: str,
    error_message: str,
    **kwargs
) -> None:
    """Log request_failed lifecycle event."""
    log_event(
        tts_logger,
        "request_failed",
        level=logging.ERROR,
        request_id=request_id,
        status_code=status_code,
        error_code=error_code,
        error_message=error_message,
        **kwargs
    )


Summary: auto


Before I modify, I’ll keep changes scoped to timing/timeout instrumentation so we can capture exact blocking stage and avoid silent hangs in benchmarks. This means: add stage-level logs + bounded wait on generation loops, with optional env toggles for timeout.


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py
@@
 import os
 import sys
 import time
+import asyncio
 import logging
 from threading import Lock
 from dataclasses import dataclass, field
@@
 def _timing_enabled() -> bool:
     """Return True when detailed request timing should be emitted."""
     return os.getenv(_PROFILE_REQUEST_LOGS, "").lower() in {"1", "true", "yes", "on"}
+
+
+def _generation_timeout_seconds() -> float:
+    """Optional hard timeout (seconds) for model generation in this request path."""
+    raw = os.getenv("VEENA3_REQUEST_TIMEOUT_SECONDS", "").strip()
+    if not raw:
+        return 0.0
+    try:
+        value = float(raw)
+        return max(0.0, value)
+    except ValueError:
+        return 0.0
@@
     try:
         logger.info(
             "tts_runtime_nonstream_start",
             extra={
@@
                 "max_tokens": max_tokens,
             },
         )
-
+        if request_timeout := _generation_timeout_seconds():
+            logger.info(
+                "tts_runtime_nonstream_generate_start",
+                extra={
+                    "speaker": speaker,
+                    "request_id": request_id,
+                    "max_tokens": max_tokens,
+                    "generation_timeout_s": request_timeout,
+                },
+            )
+
         gen_kwargs = dict(
             speaker=speaker,
             text=text,
@@
         if _timing_enabled():
-            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
-                **gen_kwargs,
-                return_timing=True,
-                request_id=request_id,
-            )
+            generation_coro = runtime.pipeline.generate_speech_indic(
+                **gen_kwargs,
+                return_timing=True,
+                request_id=request_id,
+            )
+            if request_timeout:
+                try:
+                    audio_bytes, pipeline_timings = await asyncio.wait_for(generation_coro, timeout=request_timeout)
+                except asyncio.TimeoutError as exc:
+                    raise TimeoutError(
+                        f"Nonstream generation timed out after {request_timeout:.2f}s"
+                    ) from exc
+            else:
+                audio_bytes, pipeline_timings = await generation_coro
         else:
-            audio_bytes = await runtime.pipeline.generate_speech_indic(
-                **gen_kwargs,
-                request_id=request_id,
-            )
+            generation_coro = runtime.pipeline.generate_speech_indic(
+                **gen_kwargs,
+                request_id=request_id,
+            )
+            if request_timeout:
+                try:
+                    audio_bytes = await asyncio.wait_for(generation_coro, timeout=request_timeout)
+                except asyncio.TimeoutError as exc:
+                    raise TimeoutError(
+                        f"Nonstream generation timed out after {request_timeout:.2f}s"
+                    ) from exc
+            else:
+                audio_bytes = await generation_coro
@@
     except Exception as e:
         logger.error(f"Speech generation failed: {e}")
+        logger.exception("tts_runtime_nonstream_error", extra={"speaker": speaker, "request_id": request_id})
         raise
     finally:
         active_requests = _with_inflight_delta(-1)
*** End Patch


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:
def _timing_enabled() -> bool:
    """Return True when detailed request timing should be emitted."""
    return os.getenv(_PROFILE_REQUEST_LOGS, "").lower() in {"1", "true", "yes", "on"}


sed -n '1,220p' veena3modal/services/tts_runtime.py


Output:
Chunk ID: 1ec0d2
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2039
Output:
"""
TTS runtime wrapper (container-scoped singleton).

In Modal, each container should load the model once and then handle many requests.
This wrapper owns:
- SparkTTSModel (vLLM engine)
- IndicPromptBuilder
- BiCodecDecoder
- SparkTTSPipeline + Veena3SlidingWindowPipeline
- optional SuperResolutionService

Imports framework-agnostic inference + processing code from `veena3modal/core`,
`veena3modal/processing`, and `veena3modal/audio`.
"""

from __future__ import annotations

import os
import sys
import time
import logging
from threading import Lock
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

logger = logging.getLogger(__name__)

_PROFILE_REQUEST_TIMING = "VEENA3_TIMING_LOGS"
_inflight_counter = 0
_inflight_lock = Lock()

def _timing_enabled() -> bool:
    """Return True when detailed request timing should be emitted."""
    return os.getenv(_PROFILE_REQUEST_TIMING, "").lower() in {"1", "true", "yes", "on"}

def _with_inflight_delta(delta: int) -> int:
    """Update in-flight request counter and return the new value."""
    global _inflight_counter
    with _inflight_lock:
        _inflight_counter = max(0, _inflight_counter + delta)
        return _inflight_counter

@dataclass
class TTSRuntime:
    """
    Holds long-lived, per-container inference objects.

    Thread Safety:
        This class is designed for async concurrency within a single container.
        The vLLM engine handles internal batching and scheduling.
        Do NOT share across processes without proper synchronization.
    """
    model: Any = None
    pipeline: Any = None
    streaming_pipeline: Any = None
    prompt_builder: Any = None
    bicodec_decoder: Any = None
    sr_service: Optional[Any] = None

    model_version: str = "not_loaded"
    is_loaded: bool = False
    load_time_ms: float = 0.0

    # Configuration
    model_path: str = ""
    bicodec_path: str = ""
    sr_checkpoint_dir: Optional[str] = None
    device: str = "cuda"

    # OPTIMIZATION: Pre-computed global tokens per speaker (eliminates ~110ms pre-roll in streaming)
    # Maps speaker_name -> list of 32 global token IDs captured at startup
    speaker_global_cache: Dict[str, List[int]] = field(default_factory=dict)

# Module-level singleton (per-container)
_runtime: Optional[TTSRuntime] = None

def get_runtime() -> Optional[TTSRuntime]:
    """Get the current TTS runtime singleton."""
    return _runtime

def is_initialized() -> bool:
    """Check if the runtime is initialized and ready."""
    return _runtime is not None and _runtime.is_loaded

def initialize_runtime(
    model_path: Optional[str] = None,
    bicodec_path: Optional[str] = None,
    sr_checkpoint_dir: Optional[str] = None,
    device: str = "cuda",
    hf_token: Optional[str] = None,
    gpu_memory_utilization: float = 0.25,
    enable_sr: bool = False,
    num_engines: int = 1,
) -> TTSRuntime:
    """
    Initialize the TTS runtime with all components.

    This should be called once per container (e.g., in Modal's @modal.enter).

    Args:
        model_path: Path to Spark TTS model (env: SPARK_TTS_MODEL_PATH)
        bicodec_path: Path to BiCodec model (env: BICODEC_MODEL_PATH, defaults to model_path)
        sr_checkpoint_dir: Path to super-resolution checkpoints (env: AP_BWE_CHECKPOINT_DIR)
        device: Device for inference (cuda/cpu)
        hf_token: HuggingFace token for private models (env: HF_TOKEN)
        gpu_memory_utilization: vLLM GPU memory fraction (default: 0.25)
        enable_sr: Enable super-resolution service
        num_engines: Number of vLLM engine instances (default: 1, set to 2-3 for Tier 3 optimization)

    Returns:
        Initialized TTSRuntime instance

    Raises:
        RuntimeError: If initialization fails
    """
    global _runtime

    start_time = time.time()

    # Resolve paths from env vars if not provided
    model_path = model_path or os.environ.get(
        'SPARK_TTS_MODEL_PATH',
        os.environ.get('MODEL_PATH', '/models/spark_tts_4speaker')
    )
    bicodec_path = bicodec_path or os.environ.get(
        'BICODEC_MODEL_PATH',
        model_path  # BiCodec is usually in the same directory
    )
    sr_checkpoint_dir = sr_checkpoint_dir or os.environ.get('AP_BWE_CHECKPOINT_DIR')
    hf_token = hf_token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')

    logger.info(f"Initializing TTS runtime...")
    logger.info(f"  Model path: {model_path}")
    logger.info(f"  BiCodec path: {bicodec_path}")
    logger.info(f"  SR enabled: {enable_sr}, path: {sr_checkpoint_dir}")

    try:
        # Import inference components (vendored into veena3modal for Modal deployment)
        from veena3modal.core.model_loader import SparkTTSModel
        from veena3modal.core.pipeline import SparkTTSPipeline
        from veena3modal.core.bicodec_decoder import BiCodecDecoder
        from veena3modal.processing.prompt_builder import IndicPromptBuilder

        # Load SparkTTS model with vLLM engine
        if num_engines > 1:
            # TIER 3 OPTIMIZATION: Multiple vLLM engines on same GPU
            # Each engine gets gpu_memory_utilization / num_engines to share GPU fairly.
            # E.g., 3 engines at 0.08 each = 0.24 total, ~8GB per engine, ~24GB total.
            from veena3modal.core.multi_engine import create_multi_engine_model
            per_engine_mem = gpu_memory_utilization / num_engines
            logger.info(f"Loading {num_engines} vLLM engines ({per_engine_mem:.2f} GPU mem each)...")
            model = create_multi_engine_model(
                model_path=model_path,
                num_engines=num_engines,
                hf_token=hf_token,
                gpu_memory_per_engine=per_engine_mem,
            )
        else:
            logger.info("Loading SparkTTS model with vLLM...")
            model = SparkTTSModel(
                model_path=model_path,
                hf_token=hf_token,
                gpu_memory_utilization=gpu_memory_utilization,
            )

        # Initialize prompt builder
        logger.info("Initializing prompt builder...")
        prompt_builder = IndicPromptBuilder(
            tokenizer=model.tokenizer,
            model=model,
        )

        # Initialize BiCodec decoder
        logger.info("Initializing BiCodec decoder...")
        bicodec_decoder = BiCodecDecoder(
            device=device,
            model_path=bicodec_path,
        )

        # Initialize pipeline
        logger.info("Initializing TTS pipeline...")
        pipeline = SparkTTSPipeline(
            model=model,
            prompt_builder=prompt_builder,
            bicodec_decoder=bicodec_decoder,
        )

        # Initialize streaming pipeline (for M4)
        streaming_pipeline = None
        try:
            from veena3modal.core.streaming_pipeline import Veena3SlidingWindowPipeline
            logger.info("Initializing streaming pipeline...")
            # NOTE: Parameter is named 'snac_decoder' for legacy reasons, but works with BiCodecDecoder
            streaming_pipeline = Veena3SlidingWindowPipeline(
                model=model,
                prompt_builder=prompt_builder,
                snac_decoder=bicodec_decoder,  # BiCodecDecoder is interface-compatible
            )
        except ImportError as e:
            logger.warning(f"Streaming pipeline not available: {e}")

        # Initialize super-resolution (optional)
        sr_service = None
        if enable_sr and sr_checkpoint_dir:
            try:
                from veena3modal.core.super_resolution import SuperResolutionService
                logger.info(f"Initializing super-resolution from {sr_checkpoint_dir}...")
                sr_service = SuperResolutionService(checkpoint_dir=sr_checkpoint_dir)
                # Load the model explicitly
                if sr_service.load_model(device=device):
                    logger.info("✅ Super-resolution model loaded successfully")
                else:
                    logger.warning("Super-resolution model failed to load")


rg -n "def generate_speech\(|generation_ms|tts_runtime_nonstream_start|tts_runtime_nonstream_complete|chunked_start|generate_speech_chunked\(" veena3modal/services/tts_runtime.py && sed -n '240,430p' veena3modal/services/tts_runtime.py


Output:
Chunk ID: 3158af
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 1942
Output:
365:async def generate_speech(
406:        "generation_ms": 0,
425:            "tts_runtime_nonstream_start",
462:        metrics["generation_ms"] = int(generation_time * 1000)
463:        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time
475:                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
521:                "tts_runtime_nonstream_complete",
600:async def generate_speech_chunked(
638:        "generation_ms": 0,
729:        metrics["generation_ms"] = int(generation_time * 1000)
730:        metrics["ttfb_ms"] = metrics["generation_ms"]
737:                "llm_batch_wall_ms_total": chunk_timing.get("llm_wall_ms", metrics["generation_ms"]),
971:        metrics["generation_ms"] = int((time.time() - start_time) * 1000)
981:                    "generation_ms": metrics["generation_ms"],
            model_version=model_version,
            is_loaded=True,
            load_time_ms=load_time,
            model_path=model_path,
            bicodec_path=bicodec_path,
            sr_checkpoint_dir=sr_checkpoint_dir,
            device=device,
        )

        logger.info(f"✅ TTS runtime initialized in {load_time:.0f}ms")
        logger.info(f"   Model version: {model_version}")

        # OPTIMIZATION: Pre-compute global tokens for all speakers
        # Disabled with multi-engine to avoid async event loop conflicts
        if num_engines <= 1:
            _precompute_speaker_globals(_runtime)
        else:
            logger.info("Speaker globals cache skipped (multi-engine mode)")

        # Update FastAPI app with model version
        try:
            from veena3modal.api.fastapi_app import set_model_version
            set_model_version(model_version)
        except ImportError:
            pass

        return _runtime

    except Exception as e:
        logger.error(f"❌ Failed to initialize TTS runtime: {e}")
        import traceback
        traceback.print_exc()
        raise RuntimeError(f"TTS runtime initialization failed: {e}") from e

def _precompute_speaker_globals(runtime: TTSRuntime) -> None:
    """
    Pre-compute and cache global tokens for all 12 speakers at startup.

    OPTIMIZATION: BiCodec streaming requires 32 "global tokens" before any audio can be emitted.
    These encode speaker identity via FSQ quantization. Since we have only 12 fixed speakers,
    we generate one short utterance per speaker at startup, capture the 32 global tokens,
    and cache them. During streaming, we inject cached globals via build_prefix_with_globals(),
    skipping the ~110ms global token pre-roll phase entirely.

    This runs synchronously at startup (adds ~5-10s to cold start, saves ~110ms per streaming TTFB).
    """
    import asyncio
    import re
    from vllm import SamplingParams
    from veena3modal.core.constants import (
        SPEAKER_MAP, TRAINING_STOP_TOKEN_IDS,
        DEFAULT_TEMPERATURE, DEFAULT_TOP_K, DEFAULT_TOP_P,
    )

    logger.info("Pre-computing global tokens for all speakers...")
    t_start = time.time()

    test_text = "Hello, this is a voice test."

    sampling_params = SamplingParams(
        temperature=DEFAULT_TEMPERATURE,
        top_k=DEFAULT_TOP_K,
        top_p=DEFAULT_TOP_P,
        max_tokens=128,  # Only need ~60 tokens (32 global + some semantic)
        stop=TRAINING_STOP_TOKEN_IDS,
        skip_special_tokens=False,
    )

    async def _capture_globals_for_speaker(speaker_name: str) -> Optional[List[int]]:
        """Generate a short utterance and capture the 32 global tokens."""
        prompt = runtime.prompt_builder.build_prefix(speaker_name, test_text)
        request_id = f"warmup-{speaker_name}-{int(time.time() * 1000)}"

        final_output = None
        async for request_output in runtime.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        ):
            final_output = request_output

        if final_output is None:
            return None

        generated_text = final_output.outputs[0].text
        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
        global_ids = [int(t) for t in global_matches]

        if len(global_ids) >= 32:
            return global_ids[:32]
        return None

    async def _run_all():
        for speaker_name in SPEAKER_MAP.keys():
            try:
                global_ids = await _capture_globals_for_speaker(speaker_name)
                if global_ids and len(global_ids) == 32:
                    runtime.speaker_global_cache[speaker_name] = global_ids
                    logger.info(f"  Cached globals for {speaker_name}")
                else:
                    logger.warning(f"  Failed to capture globals for {speaker_name}")
            except Exception as e:
                logger.warning(f"  Error caching globals for {speaker_name}: {e}")

    # Run async warmup in event loop
    try:
        loop = asyncio.get_event_loop()
        if loop.is_running():
            # Already in async context (unlikely at startup, but handle it)
            import concurrent.futures
            with concurrent.futures.ThreadPoolExecutor() as pool:
                pool.submit(lambda: asyncio.run(_run_all())).result()
        else:
            loop.run_until_complete(_run_all())
    except RuntimeError:
        # No event loop exists yet
        asyncio.run(_run_all())

    elapsed = (time.time() - t_start) * 1000
    cached = len(runtime.speaker_global_cache)
    total = len(SPEAKER_MAP)
    logger.info(f"Speaker globals cached: {cached}/{total} speakers in {elapsed:.0f}ms")

async def generate_speech(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    output_sample_rate: str = "16khz",
    request_id: Optional[str] = None,
) -> Tuple[Optional[bytes], Dict[str, Any]]:
    """
    Generate speech audio (non-streaming).

    Args:
        text: Text to synthesize (already normalized)
        speaker: Internal speaker name (resolved)
        temperature: Sampling temperature
        top_k: Top-k sampling
        top_p: Nucleus sampling
        max_tokens: Maximum tokens to generate
        repetition_penalty: Repetition penalty
        seed: Random seed for reproducibility
        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)

    Returns:
        Tuple of (audio_bytes, metrics_dict)
        audio_bytes: WAV audio bytes (16kHz or 48kHz, 16-bit PCM) or None if failed
        metrics_dict: Dictionary with timing metrics

    Raises:
        RuntimeError: If runtime not initialized
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()
    metrics = {
        "request_id": request_id,
        "ttfb_ms": 0,
        "generation_ms": 0,
        "tokens_generated": 0,
        "audio_duration_seconds": 0.0,
        "sr_applied": False,
        "output_sample_rate": 16000,
    }

    if request_id is None:
        import uuid
        request_id = f"runtime-ns-{uuid.uuid4().hex[:10]}"
        metrics["request_id"] = request_id

    active_requests = _with_inflight_delta(1)
    start_time = time.time()

    pipeline_timings = {}

    try:
        logger.info(
            "tts_runtime_nonstream_start",
            extra={
                "speaker": speaker,
                "request_id": request_id,
                "text_length": len(text),
                "request_inflight": active_requests,


sed -n '430,560p' veena3modal/services/tts_runtime.py && sed -n '560,840p' veena3modal/services/tts_runtime.py


Output:
Chunk ID: 894288
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 4036
Output:
                "request_inflight": active_requests,
                "temperature": temperature,
                "top_k": top_k,
                "top_p": top_p,
                "max_tokens": max_tokens,
            },
        )

        gen_kwargs = dict(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
        )

        if _timing_enabled():
            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
                **gen_kwargs,
                return_timing=True,
                request_id=request_id,
            )
        else:
            audio_bytes = await runtime.pipeline.generate_speech_indic(
                **gen_kwargs,
                request_id=request_id,
            )

        generation_time = time.time() - start_time
        metrics["generation_ms"] = int(generation_time * 1000)
        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time

        if pipeline_timings:
            metrics.update({
                "llm_token_total": pipeline_timings.get("token_total", 0),
                "llm_batch_count": pipeline_timings.get("batch_count", 0),
                "llm_batch_gpu_ms_total": pipeline_timings.get("llm_gpu_batch_ms_total", 0),
                "llm_batch_gpu_ms_min": pipeline_timings.get("llm_gpu_batch_ms_min", 0),
                "llm_batch_gpu_ms_max": pipeline_timings.get("llm_gpu_batch_ms_max", 0),
                "llm_batch_gpu_ms_p50": pipeline_timings.get("llm_gpu_batch_ms_p50", 0),
                "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
                "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
                "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
                "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
                "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
                "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
            })

        if audio_bytes:
            # Apply super-resolution if requested and available
            sample_rate = 16000
            logger.info(f"SR check: output_sample_rate={output_sample_rate!r}, "
                       f"sr_service={runtime.sr_service is not None}, "
                       f"is_loaded={runtime.sr_service.is_loaded if runtime.sr_service else False}")

            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
                try:
                    sr_start = time.time()
                    logger.info(f"Applying super-resolution to {len(audio_bytes)} bytes...")
                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
                    sr_time = (time.time() - sr_start) * 1000
                    metrics["sr_ms"] = int(sr_time)
                    metrics["sr_applied"] = True
                    sample_rate = 48000
                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
                except Exception as e:
                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
                    import traceback
                    traceback.print_exc()

            metrics["output_sample_rate"] = sample_rate

            # Calculate audio duration
            audio_duration = (len(audio_bytes) - 44) / (sample_rate * 2)  # -44 for WAV header
            metrics["audio_duration_seconds"] = max(0.0, audio_duration)

        return audio_bytes, metrics

    except Exception as e:
        logger.error(f"Speech generation failed: {e}")
        raise
    finally:
        active_requests = _with_inflight_delta(-1)
        if _timing_enabled():
            metrics["request_inflight"] = active_requests
            metrics["active_requests"] = active_requests
            logger.info(
                "tts_runtime_nonstream_complete",
                extra=metrics,
            )
        if _timing_enabled():
            logger.info(
                "tts_runtime_nonstream_inflight",
                extra={
                    "request_id": request_id,
                    "active_requests": active_requests,
                },
            )

def _apply_super_resolution(audio_bytes: bytes, sr_service) -> bytes:
    """
    Apply super-resolution to audio bytes.

    Args:
        audio_bytes: WAV audio at 16kHz
        sr_service: SuperResolutionService instance

    Returns:
        WAV audio at 48kHz
    """
    import numpy as np
    import struct
    import torch

    # Parse WAV header and extract PCM data
    if len(audio_bytes) < 44:
        raise ValueError("Invalid WAV data")

    # Extract PCM data (skip 44-byte WAV header)
    pcm_data = np.frombuffer(audio_bytes[44:], dtype=np.int16)

    # Convert to float32 for SR model
    audio_float = pcm_data.astype(np.float32) / 32768.0

    # Convert numpy to torch tensor for process_chunk
    audio_tensor = torch.from_numpy(audio_float)
    audio_tensor = torch.from_numpy(audio_float)

    # Apply super-resolution using process_chunk (16kHz -> 48kHz)
    # process_chunk expects [batch, samples] or [samples] and returns [batch, samples]
    upsampled_tensor = sr_service.process_chunk(audio_tensor)

    # Convert back to numpy, squeeze batch dim if present
    upsampled = upsampled_tensor.squeeze().cpu().numpy()

    # Convert back to int16
    upsampled_int16 = np.clip(upsampled * 32768.0, -32768, 32767).astype(np.int16)

    # Create new WAV header for 48kHz
    sample_rate = 48000
    num_channels = 1
    bits_per_sample = 16
    byte_rate = sample_rate * num_channels * bits_per_sample // 8
    block_align = num_channels * bits_per_sample // 8
    data_size = len(upsampled_int16) * 2

    wav_header = struct.pack(
        '<4sI4s4sIHHIIHH4sI',
        b'RIFF',
        36 + data_size,
        b'WAVE',
        b'fmt ',
        16,  # fmt chunk size
        1,   # PCM format
        num_channels,
        sample_rate,
        byte_rate,
        block_align,
        bits_per_sample,
        b'data',
        data_size,
    )

    return wav_header + upsampled_int16.tobytes()

async def generate_speech_chunked(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    sample_rate: int = 16000,
    output_sample_rate: str = "16khz",
    request_id: Optional[str] = None,
) -> Tuple[Optional[bytes], Dict[str, Any]]:
    """
    Generate speech with automatic text chunking for long inputs.

    Uses LongTextProcessor to split text and stitch audio.

    Args:
        text: Text to synthesize
        speaker: Internal speaker name
        ... (same as generate_speech)
        sample_rate: Internal generation sample rate (always 16000)
        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)

    Returns:
        Tuple of (audio_bytes, metrics_dict)
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()
    if request_id is None:
        import uuid
        request_id = f"runtime-chunk-{uuid.uuid4().hex[:10]}"
    metrics = {
        "request_id": request_id,
        "ttfb_ms": 0,
        "generation_ms": 0,
        "chunks_processed": 0,
        "audio_duration_seconds": 0.0,
        "text_chunked": False,
        "sr_applied": False,
        "output_sample_rate": 16000,
    }

    active_requests = _with_inflight_delta(1)
    start_time = time.time()

    try:
        if _timing_enabled():
            logger.info(
                "tts_runtime_chunking_start",
                extra={
                    "speaker": speaker,
                    "request_id": request_id,
                    "text_length": len(text),
                    "request_inflight": active_requests,
                    "chunking": True,
                },
            )
        # Import long text processor
        from veena3modal.processing.long_text_processor import LongTextProcessor

        long_processor = LongTextProcessor(pipeline=runtime.pipeline)

        # Check if chunking is needed
        if long_processor.should_chunk(text):
            metrics["text_chunked"] = True
            if _timing_enabled():
                chunk_result = await long_processor.generate_with_chunking(
                    text=text,
                    speaker=speaker,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    sample_rate=sample_rate,
                    request_id=request_id,
                    return_timing=True,
                )
                if isinstance(chunk_result, tuple):
                    audio_bytes, chunk_timing = chunk_result
                else:
                    audio_bytes = chunk_result
                    chunk_timing = {}
            else:
                audio_bytes = await long_processor.generate_with_chunking(
                    text=text,
                    speaker=speaker,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    sample_rate=sample_rate,
                    request_id=request_id,
                )
                chunk_timing = {}
        else:
            # Short text: direct generation
            gen_kwargs = dict(
                speaker=speaker,
                text=text,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
            )
            if _timing_enabled():
                audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
                    **gen_kwargs,
                    return_timing=True,
                    request_id=request_id,
                )
                chunk_timing = pipeline_timings
            else:
                audio_bytes = await runtime.pipeline.generate_speech_indic(
                    **gen_kwargs,
                    request_id=request_id,
                )
                chunk_timing = {}

        generation_time = time.time() - start_time
        metrics["generation_ms"] = int(generation_time * 1000)
        metrics["ttfb_ms"] = metrics["generation_ms"]

        if chunk_timing:
            metrics.update({
                "llm_token_total": chunk_timing.get("token_total", 0),
                "llm_batch_count": chunk_timing.get("batch_count", 0),
                "llm_batch_gpu_ms_total": chunk_timing.get("llm_gpu_batch_ms_total", 0),
                "llm_batch_wall_ms_total": chunk_timing.get("llm_wall_ms", metrics["generation_ms"]),
                "llm_parse_ms": chunk_timing.get("parse_ms", 0),
                "llm_decode_cpu_ms": chunk_timing.get("decode_cpu_ms", 0),
                "llm_batch_wall_ms_min": chunk_timing.get("llm_batch_wall_ms_min", 0),
                "llm_batch_wall_ms_max": chunk_timing.get("llm_batch_wall_ms_max", 0),
                "llm_batch_gpu_ms_min": chunk_timing.get("llm_batch_gpu_ms_min", 0),
                "llm_batch_gpu_ms_max": chunk_timing.get("llm_batch_gpu_ms_max", 0),
                "llm_batch_gpu_ms_p50": chunk_timing.get("llm_batch_gpu_ms_p50", 0),
                "tokens_per_batch_min": chunk_timing.get("tokens_per_batch_min"),
                "tokens_per_batch_max": chunk_timing.get("tokens_per_batch_max"),
                "tokens_per_batch_p50": chunk_timing.get("tokens_per_batch_p50"),
                "llm_batch_tokens": chunk_timing.get("token_batches", []),
            })
            if metrics["text_chunked"]:
                metrics["chunks_processed"] = chunk_timing.get("chunks_processed", 0)

        if audio_bytes:
            # Apply super-resolution if requested and available
            final_sample_rate = 16000
            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
                try:
                    sr_start = time.time()
                    logger.info(f"Applying super-resolution to chunked audio ({len(audio_bytes)} bytes)...")
                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
                    sr_time = (time.time() - sr_start) * 1000
                    metrics["sr_ms"] = int(sr_time)
                    metrics["sr_applied"] = True
                    final_sample_rate = 48000
                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
                except Exception as e:
                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
                    import traceback
                    traceback.print_exc()

            metrics["output_sample_rate"] = final_sample_rate
            audio_duration = (len(audio_bytes) - 44) / (final_sample_rate * 2)
            metrics["audio_duration_seconds"] = max(0.0, audio_duration)

        if _timing_enabled():
            metrics["active_requests"] = active_requests - 1
            metrics["request_inflight"] = active_requests - 1
            logger.info("tts_runtime_chunking_complete", extra=metrics)
        return audio_bytes, metrics

    except Exception as e:
        logger.error(f"Chunked speech generation failed: {e}")
        raise
    finally:
        active_requests = _with_inflight_delta(-1)
        metrics["active_requests"] = active_requests
        metrics["request_inflight"] = active_requests

from typing import AsyncGenerator

async def generate_speech_streaming(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    enable_chunking: bool = True,
    request_id: Optional[str] = None,
) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
    """
    Generate speech audio with true streaming (yields chunks as they're generated).

    This is the core streaming implementation for M4.
    First yield includes WAV header + first PCM chunk.
    Subsequent yields are raw PCM chunks.

    Args:
        text: Text to synthesize (already normalized)
        speaker: Internal speaker name (resolved)
        temperature: Sampling temperature
        top_k: Top-k sampling
        top_p: Nucleus sampling
        max_tokens: Maximum tokens to generate
        repetition_penalty: Repetition penalty
        seed: Random seed for reproducibility
        enable_chunking: Enable text chunking for long inputs with voice consistency

    Yields:
        Tuple of (audio_bytes, metrics_dict)
        - First yield: WAV header (44 bytes) prepended to first PCM chunk
        - Subsequent yields: Raw PCM chunks (int16, 16kHz)
        - Final yield: metrics_dict has final timing info

    Raises:
        RuntimeError: If runtime not initialized or streaming pipeline unavailable
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()

    if request_id is None:
        import uuid
        request_id = f"runtime-stream-{uuid.uuid4().hex[:10]}"


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py
@@
 import os
 import sys
 import time
+import asyncio
 import logging
@@
 def _timing_enabled() -> bool:
     """Return True when detailed request timing should be emitted."""
     return os.getenv(_PROFILE_REQUEST_TIMING, "").lower() in {"1", "true", "yes", "on"}

+def _generation_timeout_seconds() -> float:
+    """Optional hard timeout (seconds) for each generation call."""
+    raw = os.getenv("VEENA3_REQUEST_TIMEOUT_SECONDS", "").strip()
+    if not raw:
+        return 0.0
+    try:
+        value = float(raw)
+    except ValueError:
+        return 0.0
+    return max(0.0, value)
+
+
 def _with_inflight_delta(delta: int) -> int:
@@
         logger.info(
             "tts_runtime_nonstream_start",
             extra={
                 "speaker": speaker,
                 "request_id": request_id,
@@
                 "max_tokens": max_tokens,
             },
         )
+
+        generation_timeout = _generation_timeout_seconds()

         gen_kwargs = dict(
             speaker=speaker,
             text=text,
@@

         if _timing_enabled():
-            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
-                **gen_kwargs,
-                return_timing=True,
-                request_id=request_id,
-            )
+            generation_coro = runtime.pipeline.generate_speech_indic(
+                **gen_kwargs,
+                return_timing=True,
+                request_id=request_id,
+            )
+            if generation_timeout:
+                try:
+                    audio_bytes, pipeline_timings = await asyncio.wait_for(
+                        generation_coro,
+                        timeout=generation_timeout,
+                    )
+                except asyncio.TimeoutError as exc:
+                    raise TimeoutError(
+                        f"Non-stream generation timed out after {generation_timeout:.2f}s"
+                    ) from exc
+            else:
+                audio_bytes, pipeline_timings = await generation_coro
         else:
-            audio_bytes = await runtime.pipeline.generate_speech_indic(
-                **gen_kwargs,
-                request_id=request_id,
-            )
+            generation_coro = runtime.pipeline.generate_speech_indic(
+                **gen_kwargs,
+                request_id=request_id,
+            )
+            if generation_timeout:
+                try:
+                    audio_bytes = await asyncio.wait_for(
+                        generation_coro,
+                        timeout=generation_timeout,
+                    )
+                except asyncio.TimeoutError as exc:
+                    raise TimeoutError(
+                        f"Non-stream generation timed out after {generation_timeout:.2f}s"
+                    ) from exc
+            else:
+                audio_bytes = await generation_coro
@@
     except Exception as e:
         logger.error(f"Speech generation failed: {e}")
+        logger.exception(
+            "tts_runtime_nonstream_error",
+            extra={
+                "speaker": speaker,
+                "request_id": request_id,
+                "error": str(e),
+            },
+        )
         raise
     finally:
@@
         # Import long text processor
         from veena3modal.processing.long_text_processor import LongTextProcessor
@@
         long_processor = LongTextProcessor(pipeline=runtime.pipeline)
+        generation_timeout = _generation_timeout_seconds()
@@
             if _timing_enabled():
-                chunk_result = await long_processor.generate_with_chunking(
+                chunk_result = await long_processor.generate_with_chunking(
                     text=text,
                     speaker=speaker,
                     temperature=temperature,
@@
         else:
             # Short text: direct generation
             gen_kwargs = dict(
                 speaker=speaker,
                 text=text,
@@
             )
             if _timing_enabled():
-                audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
-                    **gen_kwargs,
-                    return_timing=True,
-                    request_id=request_id,
-                )
+                generation_coro = runtime.pipeline.generate_speech_indic(
+                    **gen_kwargs,
+                    return_timing=True,
+                    request_id=request_id,
+                )
+                if generation_timeout:
+                    try:
+                        audio_bytes, pipeline_timings = await asyncio.wait_for(
+                            generation_coro,
+                            timeout=generation_timeout,
+                        )
+                    except asyncio.TimeoutError as exc:
+                        raise TimeoutError(
+                            f"Chunked generation timed out after {generation_timeout:.2f}s"
+                        ) from exc
+                else:
+                    audio_bytes, pipeline_timings = await generation_coro
                 chunk_timing = pipeline_timings
             else:
-                audio_bytes = await runtime.pipeline.generate_speech_indic(
-                    **gen_kwargs,
-                    request_id=request_id,
-                )
+                generation_coro = runtime.pipeline.generate_speech_indic(
+                    **gen_kwargs,
+                    request_id=request_id,
+                )
+                if generation_timeout:
+                    try:
+                        audio_bytes = await asyncio.wait_for(
+                            generation_coro,
+                            timeout=generation_timeout,
+                        )
+                    except asyncio.TimeoutError as exc:
+                        raise TimeoutError(
+                            f"Chunked generation timed out after {generation_timeout:.2f}s"
+                        ) from exc
+                else:
+                    audio_bytes = await generation_coro
                 chunk_timing = {}
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


sed -n '1,170p' veena3modal/core/pipeline.py


Output:
Chunk ID: 424411
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 1383
Output:
"""
Spark TTS Generation Pipeline

End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
"""

import asyncio
import logging
import os
import re
from statistics import mean, median
from typing import Optional, List, Dict, Any, Tuple

from vllm import SamplingParams
import torch

logger = logging.getLogger(__name__)

from veena3modal.core.constants import (
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_SEED,
    AUDIO_SAMPLE_RATE,
)

def _detailed_timing_enabled() -> bool:
    """
    Return True when benchmark logs are enabled.

    Controlled by environment variable:
      VEENA3_TIMING_LOGS=true|1
    """
    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}

class SparkTTSPipeline:
    """
    End-to-end TTS pipeline for Spark TTS with BiCodec.

    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        bicodec_decoder,
    ):
        """
        Initialize pipeline.

        Args:
            model: SparkTTS Model instance
            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
            bicodec_decoder: BiCodecDecoder instance
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.bicodec_decoder = bicodec_decoder

        print(f"🚀 SparkTTSPipeline initialized")

    async def generate_speech(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
        """
        Generate speech audio (non-streaming) using Spark TTS.

        NOTE: This method signature changed from description-based to speaker-based
        to align with Spark TTS architecture.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize (with optional [emotion] tags)
            temperature: Sampling temperature (default: 0.8 for Spark TTS)
            top_k: Top-k sampling (default: 50)
            top_p: Nucleus sampling (default: 1.0)
            max_tokens: Max BiCodec tokens to generate (default: 2048)
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Build prompt using Spark TTS format
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling for Spark TTS
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
        )

        # Use vLLM engine to generate
        if request_id is None:
            import uuid
            request_id = f"req-{uuid.uuid4().hex[:12]}"

        emit_timing = return_timing or _detailed_timing_enabled()
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Collect all outputs and track generation timing
        final_output = None
        token_batches: List[int] = []
        batch_wall_ms: List[float] = []
        batch_gpu_ms: List[float] = []
        total_token_ids = 0
        first_token_ms: Optional[float] = None
        t_start = asyncio.get_event_loop().time()

        iterator = results_generator.__aiter__()
        while True:
            wall_start = asyncio.get_event_loop().time()
            start_event = None
            end_event = None
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                start_event = torch.cuda.Event(enable_timing=True)
                end_event = torch.cuda.Event(enable_timing=True)
                start_event.record()

            try:
                request_output = await iterator.__anext__()
            except StopAsyncIteration:
                break

            wall_end = asyncio.get_event_loop().time()
            batch_ms = (wall_end - wall_start) * 1000
            batch_wall_ms.append(batch_ms)

            batch_gpu_ms_value: Optional[float] = None
            if start_event is not None and end_event is not None:
                end_event.record()
                torch.cuda.synchronize()
                batch_gpu_ms_value = start_event.elapsed_time(end_event)
            if batch_gpu_ms_value is not None:
                batch_gpu_ms.append(batch_gpu_ms_value)

            total_generated = len(request_output.outputs[0].token_ids)
            new_tokens = max(0, total_generated - total_token_ids)
            total_token_ids = total_generated
            token_batches.append(new_tokens)

            if first_token_ms is None:


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/pipeline.py
@@
 def _detailed_timing_enabled() -> bool:
@@
     """
     Return True when benchmark logs are enabled.
@@
     """
     return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
+
+
+def _batch_wait_timeout_seconds() -> float:
+    """Optional hard timeout (seconds) for waiting on each vLLM iterator batch."""
+    raw = os.getenv("VEENA3_PIPELINE_BATCH_TIMEOUT_SECONDS", "").strip()
+    if not raw:
+        return 0.0
+    try:
+        return max(0.0, float(raw))
+    except ValueError:
+        return 0.0
@@
         if request_id is None:
             import uuid
             request_id = f"req-{uuid.uuid4().hex[:12]}"

         emit_timing = return_timing or _detailed_timing_enabled()
+        batch_timeout = _batch_wait_timeout_seconds()
         results_generator = self.model.engine.generate(
             prompt=prompt,
             sampling_params=sampling_params,
             request_id=request_id,
         )
@@
         first_token_ms: Optional[float] = None
         t_start = asyncio.get_event_loop().time()
+        if emit_timing:
+            logger.info(
+                "tts_llm_generate_start",
+                extra={
+                    "speaker": speaker,
+                    "request_id": request_id,
+                    "max_tokens": max_tokens,
+                    "batch_timeout_seconds": batch_timeout,
+                },
+            )

         iterator = results_generator.__aiter__()
         while True:
             wall_start = asyncio.get_event_loop().time()
@@
             if torch.cuda.is_available():
                 torch.cuda.synchronize()
                 start_event = torch.cuda.Event(enable_timing=True)
                 end_event = torch.cuda.Event(enable_timing=True)
                 start_event.record()

             try:
-                request_output = await iterator.__anext__()
+                if batch_timeout:
+                    request_output = await asyncio.wait_for(
+                        iterator.__anext__(),
+                        timeout=batch_timeout,
+                    )
+                else:
+                    request_output = await iterator.__anext__()
+            except asyncio.TimeoutError as exc:
+                if emit_timing:
+                    logger.error(
+                        "tts_llm_batch_timeout",
+                        extra={
+                            "speaker": speaker,
+                            "request_id": request_id,
+                            "batch_index": len(token_batches) + 1,
+                            "batch_timeout_seconds": batch_timeout,
+                        },
+                    )
+                raise TimeoutError(
+                    f"LLM batch wait timed out after {batch_timeout:.2f}s"
+                ) from exc
             except StopAsyncIteration:
                 break
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


I’ve added bounded-wait instrumentation in both `tts_runtime` and `SparkTTSPipeline` so each generation call logs start and times out with explicit reason if it blocks on a vLLM batch. I’ll restart the local server and rerun the one-request probe before full stress.


Output:
Chunk ID: 773d3a
Wall time: 1.0004 seconds
Process running with session ID 63481
Original token count: 83
Output:
2026-02-17 00:32:27 | INFO     | veena3modal.services.tts_runtime | tts_runtime_stream_end
^CINFO:     Shutting down
INFO:     Waiting for background tasks to complete. (CTRL+C to force quit)
ERROR 02-17 00:33:34 [core_client.py:605] Engine core proc EngineCore_DP0 died unexpectedly, shutting down client.


ps -ef | rg 'veena3modal.local_server|python3 -m veena3modal.local_server|uvicorn'


Output:
Chunk ID: 9d776d
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 126
Output:
ubuntu   3710461 3675569 56 00:30 pts/6    00:01:56 python3 -m veena3modal.local_server --skip-download --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --port 8000 --gpu-memory 0.15 --num-engines 1 --log-level INFO
ubuntu   3711850 3675569  0 00:33 ?        00:00:00 /bin/bash -lc ps -ef | rg 'veena3modal.local_server|python3 -m veena3modal.local_server|uvicorn'
ubuntu   3711855 3711850  0 00:33 ?        00:00:00 rg veena3modal.local_server|python3 -m veena3modal.local_server|uvicorn


kill -9 3710461 && sleep 1 && ps -ef | rg 'veena3modal.local_server|python3 -m veena3modal.local_server|uvicorn'


Output:
Chunk ID: 51cadc
Wall time: 0.9229 seconds
Process exited with code 0
Original token count: 76
Output:
ubuntu   3711872 3675569  0 00:33 ?        00:00:00 /bin/bash -lc kill -9 3710461 && sleep 1 && ps -ef | rg 'veena3modal.local_server|python3 -m veena3modal.local_server|uvicorn'
ubuntu   3711881 3711872  0 00:33 ?        00:00:00 rg veena3modal.local_server|python3 -m veena3modal.local_server|uvicorn


cd /home/ubuntu/veenaModal && NUMBA_DISABLE_JIT=1 PYTHONUNBUFFERED=1 VEENA3_TIMING_LOGS=true VEENA3_REQUEST_TIMEOUT_SECONDS=30 VEENA3_PIPELINE_BATCH_TIMEOUT_SECONDS=10 python3 -m veena3modal.local_server --skip-download --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --port 8000 --gpu-memory 0.15 --num-engines 1 --log-level INFO


Output:
Chunk ID: 05f32c
Wall time: 1.0015 seconds
Process running with session ID 63053
Original token count: 458
Output:
2026-02-17 00:33:43 | INFO     | veena3.local | Auth bypass enabled (local mode). Use --auth to require API keys.
2026-02-17 00:33:43 | INFO     | veena3.local | Resolved model paths:
2026-02-17 00:33:43 | INFO     | veena3.local |   LLM (vLLM):    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:33:43 | INFO     | veena3.local |   BiCodec:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:33:43 | INFO     | veena3.local | ============================================================
2026-02-17 00:33:43 | INFO     | veena3.local | Initializing Veena3 TTS Runtime (local)
2026-02-17 00:33:43 | INFO     | veena3.local |   Model:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:33:43 | INFO     | veena3.local |   LLM path:    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:33:43 | INFO     | veena3.local |   BiCodec:     /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:33:43 | INFO     | veena3.local |   Device:      cuda
2026-02-17 00:33:43 | INFO     | veena3.local |   GPU Memory:  15% (total)
2026-02-17 00:33:43 | INFO     | veena3.local |   Engines:     1
2026-02-17 00:33:43 | INFO     | veena3.local |   SR:          disabled
2026-02-17 00:33:43 | INFO     | veena3.local | ============================================================
2026-02-17 00:33:43 | INFO     | veena3modal.services.tts_runtime | Initializing TTS runtime...
2026-02-17 00:33:43 | INFO     | veena3modal.services.tts_runtime |   Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:33:43 | INFO     | veena3modal.services.tts_runtime |   BiCodec path: /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:33:43 | INFO     | veena3modal.services.tts_runtime |   SR enabled: False, path: None


Output:
Chunk ID: 6ad44d
Wall time: 5.0007 seconds
Process running with session ID 63053
Original token count: 1295
Output:
2026-02-17 00:33:48 | INFO     | veena3modal.services.tts_runtime | Loading SparkTTS model with vLLM...
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.15
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
INFO 02-17 00:33:49 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 00:33:49 [model.py:1561] Using max model len 4096
INFO 02-17 00:33:49 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 00:33:49 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3712012) INFO 02-17 00:33:50 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}


Output:
Chunk ID: dfc6de
Wall time: 10.0019 seconds
Process running with session ID 63053
Original token count: 514
Output:
(EngineCore_DP0 pid=3712012) INFO 02-17 00:33:53 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:55355 backend=nccl
(EngineCore_DP0 pid=3712012) INFO 02-17 00:33:53 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3712012) INFO 02-17 00:33:53 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=3712012) INFO 02-17 00:33:54 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=3712012) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=3712012) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=3712012)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=3712012)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.44s/it]
(EngineCore_DP0 pid=3712012)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.44s/it]
(EngineCore_DP0 pid=3712012)
(EngineCore_DP0 pid=3712012) INFO 02-17 00:33:56 [default_loader.py:291] Loading weights took 1.51 seconds
(EngineCore_DP0 pid=3712012) INFO 02-17 00:33:56 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 2.029414 seconds
(EngineCore_DP0 pid=3712012) INFO 02-17 00:34:01 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=3712012) INFO 02-17 00:34:01 [backends.py:872] Dynamo bytecode transform time: 4.22 s


Output:
Chunk ID: a8f3c0
Wall time: 10.0006 seconds
Process running with session ID 63053
Original token count: 755
Output:
(EngineCore_DP0 pid=3712012) INFO 02-17 00:34:04 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.670 s
(EngineCore_DP0 pid=3712012) INFO 02-17 00:34:04 [monitor.py:34] torch.compile takes 4.89 s in total
(EngineCore_DP0 pid=3712012) INFO 02-17 00:34:05 [gpu_worker.py:356] Available KV cache memory: 10.13 GiB
(EngineCore_DP0 pid=3712012) INFO 02-17 00:34:05 [kv_cache_utils.py:1307] GPU KV cache size: 885,184 tokens
(EngineCore_DP0 pid=3712012) INFO 02-17 00:34:05 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 216.11x
(EngineCore_DP0 pid=3712012)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%| | 0/35 [00:00<?,
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%| | 4/35 [00:00<00:
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|▏| 8/35 [00:00<00:
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|▎| 12/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|▍| 16/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|▌| 20/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|▋| 24/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|▊| 28/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|▉| 32/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|█| 35/35 [00:01<00
(EngineCore_DP0 pid=3712012)
Capturing CUDA graphs (decode, FULL):   0%|              | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|▉     | 3/19 [00:00<00:00, 26.23it/s]
Capturing CUDA graphs (decode, FULL):  37%|██▏   | 7/19 [00:00<00:00, 32.36it/s]
Capturing CUDA graphs (decode, FULL):  58%|██▉  | 11/19 [00:00<00:00, 34.49it/s]
Capturing CUDA graphs (decode, FULL):  79%|███▉ | 15/19 [00:00<00:00, 35.47it/s]
Capturing CUDA graphs (decode, FULL): 100%|█████| 19/19 [00:00<00:00, 36.25it/s]
Capturing CUDA graphs (decode, FULL): 100%|█████| 19/19 [00:00<00:00, 34.84it/s]
(EngineCore_DP0 pid=3712012) INFO 02-17 00:34:07 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=3712012) INFO 02-17 00:34:07 [core.py:272] init engine (profile, create kv cache, warmup model) took 11.04 seconds
✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

2026-02-17 00:34:09 | INFO     | veena3modal.services.tts_runtime | Initializing prompt builder...
2026-02-17 00:34:09 | INFO     | veena3modal.services.tts_runtime | Initializing BiCodec decoder...
🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cuda
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


Output:
Chunk ID: f25e15
Wall time: 5.0011 seconds
Process running with session ID 63053
Original token count: 390
Output:
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
2026-02-17 00:34:16 | INFO     | veena3modal.services.tts_runtime | Initializing TTS pipeline...
🚀 SparkTTSPipeline initialized
2026-02-17 00:34:16 | INFO     | veena3modal.services.tts_runtime | Initializing streaming pipeline...
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)
2026-02-17 00:34:16 | INFO     | veena3modal.services.tts_runtime | ✅ TTS runtime initialized in 33232ms
2026-02-17 00:34:16 | INFO     | veena3modal.services.tts_runtime |    Model version: LLM
2026-02-17 00:34:16 | INFO     | veena3modal.services.tts_runtime | Pre-computing global tokens for all speakers...
2026-02-17 00:34:17 | INFO     | veena3modal.services.tts_runtime |   Cached globals for lipakshi
2026-02-17 00:34:17 | INFO     | veena3modal.services.tts_runtime |   Cached globals for vardan
2026-02-17 00:34:18 | INFO     | veena3modal.services.tts_runtime |   Cached globals for reet
2026-02-17 00:34:18 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nandini
2026-02-17 00:34:18 | INFO     | veena3modal.services.tts_runtime |   Cached globals for krishna
2026-02-17 00:34:19 | INFO     | veena3modal.services.tts_runtime |   Cached globals for anika
2026-02-17 00:34:19 | INFO     | veena3modal.services.tts_runtime |   Cached globals for adarsh
2026-02-17 00:34:20 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nilay


The current TTS application here is an implementation in Modal labs. Which has autoscaling and other rules self scaling rules and everything suitable for my target audience. Now, i need you to create a local entry point or implementation point. then we will plan for optimising the pipeline, then we will discuss on stress testing, then we will discuss on scaling laws post optimisations locally. @veena3modal/app.py

the above part is performed by a different agent. and benchmarks are made. Now can you do the benchmarks on this newly optimized path to understand the concurrency and stress testing.

Now i need you to capture detailed time logs for each request happening in the GPU. like what happens with 1req, time per LLM token, time per batch decoding, and everything else. detailed time about where we are spending each minute in the GPU per generation.

and when you have the details for per request. you can obviously add more logging to get details of concurrency. so you understand what's the bottleneck for the peak concurrency. and then we will decide on effective methods in optimising this TTS engine.

Warning: apply_patch was requested via exec_command. Use the apply_patch tool instead of exec_command.

<permissions instructions>
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `danger-full-access`: No filesystem sandboxing - all commands are permitted. Network access is enabled.
Approval policy is currently never. Do not provide the `sandbox_permissions` for any reason, commands will be rejected.
</permissions instructions>

<collaboration_mode># Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.
</collaboration_mode>

# AGENTS.md instructions for /home/ubuntu/veenaModal

<INSTRUCTIONS>
## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.
</INSTRUCTIONS>

<environment_context>
  <cwd>/home/ubuntu/veenaModal</cwd>
  <shell>bash</shell>
</environment_context>


I’m going to run the local benchmark path end-to-end on the optimized implementation, capture per-request timing headers and GPU snapshots, then run a small stress sweep so we can identify which phase dominates under concurrency (LLM decode, batching, or post-processing). I’ll start by checking whether the local service is running, then execute the benchmark script with single and concurrent workloads and summarize the GPU/request timing breakdown.


ps -ef | rg -n "uvicorn|python .*app.py|veena3modal|scripts"


Output:
Chunk ID: 06c851
Wall time: 0.0503 seconds
Process exited with code 0
Original token count: 708
Output:
280:ubuntu   3524452 3429066  0 Feb16 pts/1    00:00:00 /bin/bash --init-file /home/ubuntu/.cursor-server/bin/linux-x64/f3f5cec40024283013878b50c4f9be4002e0b580/out/vs/workbench/contrib/terminal/common/scripts/shellIntegration-bash.sh
285:ubuntu   3567393 3429066  0 Feb16 pts/2    00:00:00 /bin/bash --init-file /home/ubuntu/.cursor-server/bin/linux-x64/f3f5cec40024283013878b50c4f9be4002e0b580/out/vs/workbench/contrib/terminal/common/scripts/shellIntegration-bash.sh
292:ubuntu   3643269 3429066  0 Feb16 pts/3    00:00:00 /bin/bash --init-file /home/ubuntu/.cursor-server/bin/linux-x64/f3f5cec40024283013878b50c4f9be4002e0b580/out/vs/workbench/contrib/terminal/common/scripts/shellIntegration-bash.sh
293:ubuntu   3644036 3429066  0 Feb16 pts/5    00:00:00 /bin/bash --init-file /home/ubuntu/.cursor-server/bin/linux-x64/f3f5cec40024283013878b50c4f9be4002e0b580/out/vs/workbench/contrib/terminal/common/scripts/shellIntegration-bash.sh
341:ubuntu   3708597 3675569  0 00:28 ?        00:00:00 /bin/bash -lc cd /home/ubuntu/veenaModal && python3 - <<'PY' import os, asyncio, sys sys.path.insert(0, os.getcwd()) os.environ['VEENA3_TIMING_LOGS']='true' os.environ['NUMBA_DISABLE_JIT']='1'  from veena3modal.services import tts_runtime as tr from veena3modal.services.tts_runtime import initialize_runtime  async def main():     # local server already initialized runtime; try to use it if available     print('initialized?', tr.is_initialized())     if not tr.is_initialized():         print('init needed')         initialize_runtime(             model_path='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM',             bicodec_path='/home/ubuntu/veenaModal/models/spark_tts_4speaker',             device='cuda',             gpu_memory_utilization=0.15,             num_engines=1,         )     try:         result = await tr.generate_speech(             text='Hello there',             speaker='lipakshi',             temperature=0.8,             top_k=50,             top_p=1.0,             max_tokens=128,             repetition_penalty=1.05,             seed=None,             output_sample_rate='16khz',             request_id='direct-test',         )         print('done', type(result[0]), len(result[0]) if result else None, isinstance(result, tuple))     except Exception as e:         print('error', type(e).__name__, e)  asyncio.run(main()) PY
345:ubuntu   3711892 3675569 99 00:33 pts/7    00:01:30 python3 -m veena3modal.local_server --skip-download --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --port 8000 --gpu-memory 0.15 --num-engines 1 --log-level INFO
374:ubuntu   3712363 3675569  0 00:34 ?        00:00:00 /bin/bash -lc ps -ef | rg -n "uvicorn|python .*app.py|veena3modal|scripts"
376:ubuntu   3712368 3712363  0 00:34 ?        00:00:00 rg -n uvicorn|python .*app.py|veena3modal|scripts


cd /home/ubuntu/veenaModal && python3 scripts/stress_test_optimized_local.py --help


Output:
Chunk ID: 8592c1
Wall time: 0.0682 seconds
Process exited with code 0
Original token count: 237
Output:
usage: stress_test_optimized_local.py [-h] [--url URL] [--levels LEVELS]
                                      [--text {short,medium,long,mixed}]
                                      [--chunking] [--no-chunking]
                                      [--max-tokens MAX_TOKENS] [--stream]
                                      [--output OUTPUT]

Detailed local optimized-path TTS benchmark

options:
  -h, --help            show this help message and exit
  --url URL             Server URL (default: http://localhost:8000)
  --levels LEVELS       Comma-separated concurrency levels
  --text {short,medium,long,mixed}
                        Input text category
  --chunking            Enable server-side chunking
  --no-chunking         Disable chunking
  --max-tokens MAX_TOKENS
                        max_tokens passed to /v1/tts/generate (default: 256)
  --stream              Use streaming endpoint
  --output OUTPUT       Output JSON path


Next I’m running a controlled single-concurrency baseline first (non-streaming, no chunking, deterministic short input) to capture per-request GPU timing headers and verify that response timing fields are present before load scaling.


cd /home/ubuntu/veenaModal && python3 scripts/stress_test_optimized_local.py --url http://localhost:8000 --levels 1 --text short --max-tokens 128 --no-chunking --output /tmp/stress_1_nonstream.json


Output:
Chunk ID: 217c9d
Wall time: 30.0010 seconds
Process running with session ID 20041
Original token count: 0
Output:


Output:
Chunk ID: b178e1
Wall time: 30.0010 seconds
Process running with session ID 20041
Original token count: 0
Output:


Output:
Chunk ID: de5fba
Wall time: 30.0008 seconds
Process running with session ID 20041
Original token count: 0
Output:


python3 - <<'PY'
import requests, time
u='http://localhost:8000/health'
for i in range(3):
    try:
        t=time.time();
        r=requests.get(u,timeout=10)
        print('status',r.status_code,'time',time.time()-t,r.text[:120])
    except Exception as e:
        print('err',e)
    time.sleep(1)
PY