Output:
Chunk ID: 9edcc4
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2188
Output:
#!/usr/bin/env python3
"""
Detailed Pipeline Profiler for Veena3 TTS.

Instruments every stage of the TTS pipeline with microsecond precision:
- Prompt building
- vLLM prefill (prompt encoding)
- vLLM decode (per-token generation)
- Token parsing (BiCodec extraction)
- BiCodec audio decode (GPU)
- Crossfade / audio post-processing
- WAV header creation

For streaming: tracks token-by-token timing to find bottleneck.
For concurrency: measures contention points under parallel load.

Usage:
    # Single request breakdown
    python scripts/profile_pipeline.py

    # Concurrency profiling (1, 5, 10, 20 parallel requests)
    python scripts/profile_pipeline.py --concurrency 1,5,10,20

    # Long text profiling
    python scripts/profile_pipeline.py --text long

    # Streaming profiling
    python scripts/profile_pipeline.py --stream
"""

from __future__ import annotations

import argparse
import asyncio
import logging
import os
import re
import sys
import time
import uuid
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

# === Path setup (same as local_server.py) ===
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
EXTERNAL_DIR = os.path.join(REPO_ROOT, "external")
for path in [os.path.join(EXTERNAL_DIR, "sparktts"), os.path.join(EXTERNAL_DIR, "AP-BWE"), REPO_ROOT]:
    if path not in sys.path:
        sys.path.insert(0, path)

os.environ["AUTH_BYPASS_MODE"] = "true"

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s | %(levelname)-8s | %(message)s",
    datefmt="%H:%M:%S",
)
logger = logging.getLogger("profiler")

# Test texts
TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This sentence tests basic text normalization and generation."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast long shadows "
        "over crowded streets, there lived an old bookkeeper named Margaret. She had spent "
        "forty years cataloging the stories of others, yet never found time to write her own. "
        "One rainy Tuesday afternoon, she discovered a leather-bound journal tucked behind "
        "a shelf of dusty encyclopedias. Its pages were blank, waiting."
    ),
}

# === Timing Utilities ===

@dataclass
class StageTimer:
    """Accumulated timing for a named pipeline stage."""
    name: str
    calls: int = 0
    total_ms: float = 0.0
    min_ms: float = float("inf")
    max_ms: float = 0.0
    samples: List[float] = field(default_factory=list)

    def record(self, ms: float):
        self.calls += 1
        self.total_ms += ms
        self.min_ms = min(self.min_ms, ms)
        self.max_ms = max(self.max_ms, ms)
        self.samples.append(ms)

    @property
    def avg_ms(self) -> float:
        return self.total_ms / self.calls if self.calls > 0 else 0

    def report_line(self) -> str:
        if self.calls == 0:
            return f"  {self.name:<35} (not called)"
        if self.calls == 1:
            return f"  {self.name:<35} {self.total_ms:>8.2f}ms"
        return (
            f"  {self.name:<35} {self.total_ms:>8.2f}ms total  "
            f"avg={self.avg_ms:.2f}ms  min={self.min_ms:.2f}ms  "
            f"max={self.max_ms:.2f}ms  calls={self.calls}"
        )

@dataclass
class RequestProfile:
    """Complete timing profile for one TTS request."""
    request_id: str
    text: str
    speaker: str
    stream: bool
    stages: Dict[str, StageTimer] = field(default_factory=dict)

    # Metadata filled during profiling
    total_ms: float = 0.0
    prompt_tokens: int = 0
    generated_tokens: int = 0
    semantic_tokens: int = 0
    global_tokens: int = 0
    audio_bytes: int = 0
    audio_seconds: float = 0.0

    # Streaming-specific
    ttfb_ms: float = 0.0
    chunks_emitted: int = 0
    token_generation_rates: List[float] = field(default_factory=list)  # tokens/sec samples
    decode_latencies: List[float] = field(default_factory=list)  # ms per BiCodec decode

    def stage(self, name: str) -> StageTimer:
        if name not in self.stages:
            self.stages[name] = StageTimer(name=name)
        return self.stages[name]

    def report(self) -> str:
        lines = [
            f"\n{'=' * 80}",
            f"  REQUEST PROFILE: {self.request_id}",
            f"  text={len(self.text)} chars, speaker={self.speaker}, stream={self.stream}",
            f"{'=' * 80}",
            f"  TOTAL: {self.total_ms:.1f}ms",
            f"  Generated: {self.generated_tokens} tokens ({self.semantic_tokens} semantic, {self.global_tokens} global)",
            f"  Audio: {self.audio_seconds:.2f}s ({self.audio_bytes} bytes)",
        ]
        if self.stream:
            lines.append(f"  TTFB: {self.ttfb_ms:.1f}ms, Chunks: {self.chunks_emitted}")
        if self.generated_tokens > 0 and self.total_ms > 0:
            tps = self.generated_tokens / (self.total_ms / 1000)
            lines.append(f"  Token gen rate: {tps:.1f} tok/s")
        if self.audio_seconds > 0:
            rtf = (self.total_ms / 1000) / self.audio_seconds
            lines.append(f"  RTF: {rtf:.3f}")

        lines.append(f"\n  {'STAGE BREAKDOWN':}")
        lines.append(f"  {'-' * 76}")

        # Order stages by total time (descending)
        sorted_stages = sorted(self.stages.values(), key=lambda s: s.total_ms, reverse=True)
        for s in sorted_stages:
            lines.append(s.report_line())

        # Token generation rate over time (streaming)
        if self.token_generation_rates:
            lines.append(f"\n  TOKEN GEN RATE (tok/s over time):")
            # Show first 5, middle, last 5
            rates = self.token_generation_rates
            if len(rates) <= 12:
                lines.append(f"    {' → '.join(f'{r:.0f}' for r in rates)}")
            else:
                first = ' → '.join(f'{r:.0f}' for r in rates[:5])
                last = ' → '.join(f'{r:.0f}' for r in rates[-5:])
                lines.append(f"    {first} → ... → {last}")
            import statistics
            lines.append(f"    avg={statistics.mean(rates):.0f}  min={min(rates):.0f}  max={max(rates):.0f}")

        # BiCodec decode latencies
        if self.decode_latencies:
            lines.append(f"\n  BICODEC DECODE LATENCIES (ms per call):")
            import statistics
            lines.append(
                f"    avg={statistics.mean(self.decode_latencies):.2f}  "
                f"min={min(self.decode_latencies):.2f}  "
                f"max={max(self.decode_latencies):.2f}  "
                f"calls={len(self.decode_latencies)}"
            )

        lines.append(f"{'=' * 80}")
        return "\n".join(lines)

# === Non-Streaming Profiler ===

async def profile_non_streaming(
    runtime,
    text: str,
    speaker: str = "lipakshi",
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
) -> RequestProfile:
    """Profile a single non-streaming request with stage-level timing."""
    from vllm import SamplingParams
    from veena3modal.core.constants import TRAINING_STOP_TOKEN_IDS, AUDIO_SAMPLE_RATE
    from veena3modal.audio.utils import add_wav_header

    profile = RequestProfile(
        request_id=f"prof-{uuid.uuid4().hex[:8]}",
        text=text,
        speaker=speaker,
        stream=False,
    )

    t_total_start = time.perf_counter()

    # === Stage 1: Prompt Building ===
    t0 = time.perf_counter()
    prompt = runtime.prompt_builder.build_prefix(speaker, text)
    profile.stage("1_prompt_build").record((time.perf_counter() - t0) * 1000)
    profile.prompt_tokens = len(runtime.model.tokenizer.encode(prompt, add_special_tokens=False))

    # === Stage 2: Sampling Config ===
    t0 = time.perf_counter()
    sampling_params = SamplingParams(
        temperature=temperature,
        top_k=top_k,
        top_p=top_p,
        max_tokens=max_tokens,
        repetition_penalty=repetition_penalty,
        stop=TRAINING_STOP_TOKEN_IDS,
        skip_special_tokens=False,
        seed=seed,
    )
    profile.stage("2_sampling_config").record((time.perf_counter() - t0) * 1000)

    # === Stage 3: vLLM Token Generation ===
    request_id = f"prof-{uuid.uuid4().hex[:8]}"
    t_gen_start = time.perf_counter()
    t_first_output = None
    iteration_count = 0
    prev_token_count = 0

    results_generator = runtime.model.engine.generate(
        prompt=prompt,
        sampling_params=sampling_params,
        request_id=request_id,
    )

    final_output = None
    async for request_output in results_generator:
        iteration_count += 1
        now = time.perf_counter()


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


I found a hard blocker before stress runs: BiCodec OOM when instantiated on GPU with current free VRAM. I’m patching the runtime to allow forcing BiCodec onto CPU (`BICODEC_DEVICE=cpu`) so we can still benchmark LLM GPU timing and request concurrency.


apply_patch <<'PATCH'
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
 def _generation_timeout_seconds() -> float:
@@
     except ValueError:
         return 0.0
     return max(0.0, value)
+
+
+def _bicodec_device(default_device: str = "cuda") -> str:
+    """Resolve BiCodec device from explicit override and env var."""
+    raw = os.getenv("BICODEC_DEVICE", "").strip().lower()
+    if raw in {"", "default"}:
+        return default_device
+    if raw not in {"cuda", "cpu"}:
+        return default_device
+    return raw
@@
 def initialize_runtime(
@@
     enable_sr: bool = False,
     num_engines: int = 1,
+    bicodec_device: Optional[str] = None,
 ) -> TTSRuntime:
@@
     try:
         # Resolve paths from env vars if not provided
         model_path = model_path or os.environ.get(
             'SPARK_TTS_MODEL_PATH',
             os.environ.get('MODEL_PATH', '/models/spark_tts_4speaker')
         )
         bicodec_path = bicodec_path or os.environ.get(
             'BICODEC_MODEL_PATH',
             model_path  # BiCodec is usually in the same directory
         )
         sr_checkpoint_dir = sr_checkpoint_dir or os.environ.get('AP_BWE_CHECKPOINT_DIR')
         hf_token = hf_token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')
+        resolved_bicodec_device = _bicodec_device(default_device=device)
+        if bicodec_device:
+            resolved_bicodec_device = bicodec_device

         logger.info(f"Initializing TTS runtime...")
         logger.info(f"  Model path: {model_path}")
         logger.info(f"  BiCodec path: {bicodec_path}")
+        logger.info(f"  BiCodec device: {resolved_bicodec_device}")
         logger.info(f"  SR enabled: {enable_sr}, path: {sr_checkpoint_dir}")
@@
         # Initialize BiCodec decoder
         logger.info("Initializing BiCodec decoder...")
         bicodec_decoder = BiCodecDecoder(
-            device=device,
+            device=resolved_bicodec_device,
             model_path=bicodec_path,
         )
*** End Patch
PATCH


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:
    try:
        # Resolve paths from env vars if not provided
        model_path = model_path or os.environ.get(
            'SPARK_TTS_MODEL_PATH',
            os.environ.get('MODEL_PATH', '/models/spark_tts_4speaker')
        )
        bicodec_path = bicodec_path or os.environ.get(
            'BICODEC_MODEL_PATH',
            model_path  # BiCodec is usually in the same directory
        )
        sr_checkpoint_dir = sr_checkpoint_dir or os.environ.get('AP_BWE_CHECKPOINT_DIR')
        hf_token = hf_token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')

        logger.info(f"Initializing TTS runtime...")
        logger.info(f"  Model path: {model_path}")
        logger.info(f"  BiCodec path: {bicodec_path}")
        logger.info(f"  SR enabled: {enable_sr}, path: {sr_checkpoint_dir}")


Summary: auto


sed -n '1,210p' veena3modal/services/tts_runtime.py


Output:
Chunk ID: 3ae9a6
Wall time: 0.0503 seconds
Process exited with code 0
Original token count: 1802
Output:
"""
TTS runtime wrapper (container-scoped singleton).

In Modal, each container should load the model once and then handle many requests.
This wrapper owns:
- SparkTTSModel (vLLM engine)
- IndicPromptBuilder
- BiCodecDecoder
- SparkTTSPipeline + Veena3SlidingWindowPipeline
- optional SuperResolutionService

Imports framework-agnostic inference + processing code from `veena3modal/core`,
`veena3modal/processing`, and `veena3modal/audio`.
"""

from __future__ import annotations

import os
import sys
import time
import asyncio
import logging
from threading import Lock
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

logger = logging.getLogger(__name__)

_PROFILE_REQUEST_TIMING = "VEENA3_TIMING_LOGS"
_inflight_counter = 0
_inflight_lock = Lock()

def _timing_enabled() -> bool:
    """Return True when detailed request timing should be emitted."""
    return os.getenv(_PROFILE_REQUEST_TIMING, "").lower() in {"1", "true", "yes", "on"}

def _generation_timeout_seconds() -> float:
    """Optional hard timeout (seconds) for each generation call."""
    raw = os.getenv("VEENA3_REQUEST_TIMEOUT_SECONDS", "").strip()
    if not raw:
        return 0.0
    try:
        value = float(raw)
    except ValueError:
        return 0.0
    return max(0.0, value)

def _with_inflight_delta(delta: int) -> int:
    """Update in-flight request counter and return the new value."""
    global _inflight_counter
    with _inflight_lock:
        _inflight_counter = max(0, _inflight_counter + delta)
        return _inflight_counter

@dataclass
class TTSRuntime:
    """
    Holds long-lived, per-container inference objects.

    Thread Safety:
        This class is designed for async concurrency within a single container.
        The vLLM engine handles internal batching and scheduling.
        Do NOT share across processes without proper synchronization.
    """
    model: Any = None
    pipeline: Any = None
    streaming_pipeline: Any = None
    prompt_builder: Any = None
    bicodec_decoder: Any = None
    sr_service: Optional[Any] = None

    model_version: str = "not_loaded"
    is_loaded: bool = False
    load_time_ms: float = 0.0

    # Configuration
    model_path: str = ""
    bicodec_path: str = ""
    sr_checkpoint_dir: Optional[str] = None
    device: str = "cuda"

    # OPTIMIZATION: Pre-computed global tokens per speaker (eliminates ~110ms pre-roll in streaming)
    # Maps speaker_name -> list of 32 global token IDs captured at startup
    speaker_global_cache: Dict[str, List[int]] = field(default_factory=dict)

# Module-level singleton (per-container)
_runtime: Optional[TTSRuntime] = None

def get_runtime() -> Optional[TTSRuntime]:
    """Get the current TTS runtime singleton."""
    return _runtime

def is_initialized() -> bool:
    """Check if the runtime is initialized and ready."""
    return _runtime is not None and _runtime.is_loaded

def initialize_runtime(
    model_path: Optional[str] = None,
    bicodec_path: Optional[str] = None,
    sr_checkpoint_dir: Optional[str] = None,
    device: str = "cuda",
    hf_token: Optional[str] = None,
    gpu_memory_utilization: float = 0.25,
    enable_sr: bool = False,
    num_engines: int = 1,
) -> TTSRuntime:
    """
    Initialize the TTS runtime with all components.

    This should be called once per container (e.g., in Modal's @modal.enter).

    Args:
        model_path: Path to Spark TTS model (env: SPARK_TTS_MODEL_PATH)
        bicodec_path: Path to BiCodec model (env: BICODEC_MODEL_PATH, defaults to model_path)
        sr_checkpoint_dir: Path to super-resolution checkpoints (env: AP_BWE_CHECKPOINT_DIR)
        device: Device for inference (cuda/cpu)
        hf_token: HuggingFace token for private models (env: HF_TOKEN)
        gpu_memory_utilization: vLLM GPU memory fraction (default: 0.25)
        enable_sr: Enable super-resolution service
        num_engines: Number of vLLM engine instances (default: 1, set to 2-3 for Tier 3 optimization)

    Returns:
        Initialized TTSRuntime instance

    Raises:
        RuntimeError: If initialization fails
    """
    global _runtime

    start_time = time.time()

    # Resolve paths from env vars if not provided
    model_path = model_path or os.environ.get(
        'SPARK_TTS_MODEL_PATH',
        os.environ.get('MODEL_PATH', '/models/spark_tts_4speaker')
    )
    bicodec_path = bicodec_path or os.environ.get(
        'BICODEC_MODEL_PATH',
        model_path  # BiCodec is usually in the same directory
    )
    sr_checkpoint_dir = sr_checkpoint_dir or os.environ.get('AP_BWE_CHECKPOINT_DIR')
    hf_token = hf_token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')

    logger.info(f"Initializing TTS runtime...")
    logger.info(f"  Model path: {model_path}")
    logger.info(f"  BiCodec path: {bicodec_path}")
    logger.info(f"  SR enabled: {enable_sr}, path: {sr_checkpoint_dir}")

    try:
        # Import inference components (vendored into veena3modal for Modal deployment)
        from veena3modal.core.model_loader import SparkTTSModel
        from veena3modal.core.pipeline import SparkTTSPipeline
        from veena3modal.core.bicodec_decoder import BiCodecDecoder
        from veena3modal.processing.prompt_builder import IndicPromptBuilder

        # Load SparkTTS model with vLLM engine
        if num_engines > 1:
            # TIER 3 OPTIMIZATION: Multiple vLLM engines on same GPU
            # Each engine gets gpu_memory_utilization / num_engines to share GPU fairly.
            # E.g., 3 engines at 0.08 each = 0.24 total, ~8GB per engine, ~24GB total.
            from veena3modal.core.multi_engine import create_multi_engine_model
            per_engine_mem = gpu_memory_utilization / num_engines
            logger.info(f"Loading {num_engines} vLLM engines ({per_engine_mem:.2f} GPU mem each)...")
            model = create_multi_engine_model(
                model_path=model_path,
                num_engines=num_engines,
                hf_token=hf_token,
                gpu_memory_per_engine=per_engine_mem,
            )
        else:
            logger.info("Loading SparkTTS model with vLLM...")
            model = SparkTTSModel(
                model_path=model_path,
                hf_token=hf_token,
                gpu_memory_utilization=gpu_memory_utilization,
            )

        # Initialize prompt builder
        logger.info("Initializing prompt builder...")
        prompt_builder = IndicPromptBuilder(
            tokenizer=model.tokenizer,
            model=model,
        )

        # Initialize BiCodec decoder
        logger.info("Initializing BiCodec decoder...")
        bicodec_decoder = BiCodecDecoder(
            device=device,
            model_path=bicodec_path,
        )

        # Initialize pipeline
        logger.info("Initializing TTS pipeline...")
        pipeline = SparkTTSPipeline(
            model=model,
            prompt_builder=prompt_builder,
            bicodec_decoder=bicodec_decoder,
        )

        # Initialize streaming pipeline (for M4)
        streaming_pipeline = None
        try:


apply_patch <<'PATCH'
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
 def _generation_timeout_seconds() -> float:
@@
     except ValueError:
         return 0.0
     return max(0.0, value)
+
+
+def _bicodec_device(default_device: str = "cuda") -> str:
+    """Resolve BiCodec device from env override and return the effective device."""
+    raw = os.getenv("BICODEC_DEVICE", "").strip().lower()
+    if not raw or raw == "default":
+        return default_device
+    if raw not in {"cuda", "cpu"}:
+        return default_device
+    return raw
@@
 def initialize_runtime(
@@
     hf_token: Optional[str] = None,
     gpu_memory_utilization: float = 0.25,
     enable_sr: bool = False,
     num_engines: int = 1,
+    bicodec_device: Optional[str] = None,
 ) -> TTSRuntime:
@@
     model_path = model_path or os.environ.get(
         'SPARK_TTS_MODEL_PATH',
         os.environ.get('MODEL_PATH', '/models/spark_tts_4speaker')
     )
+    bicodec_device = bicodec_device or _bicodec_device(default_device=device)
     bicodec_path = bicodec_path or os.environ.get(
         'BICODEC_MODEL_PATH',
         model_path  # BiCodec is usually in the same directory
     )
@@
     logger.info(f"Initializing TTS runtime...")
     logger.info(f"  Model path: {model_path}")
     logger.info(f"  BiCodec path: {bicodec_path}")
+    logger.info(f"  BiCodec device: {bicodec_device}")
     logger.info(f"  SR enabled: {enable_sr}, path: {sr_checkpoint_dir}")
@@
         logger.info("Initializing BiCodec decoder...")
         bicodec_decoder = BiCodecDecoder(
-            device=device,
+            device=bicodec_device,
             model_path=bicodec_path,
         )
*** End Patch
PATCH


Warning: apply_patch was requested via exec_command. Use the apply_patch tool instead of exec_command.


Output: {"output":"Success. Updated the following files:\nM veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


cd /home/ubuntu/veenaModal && VEENA3_TIMING_LOGS=true NUMBA_DISABLE_JIT=1 BICODEC_DEVICE=cpu GPU_MEMORY_UTILIZATION=0.1 MODEL_PATH=$(pwd)/models/spark_tts_4speaker python3 -m veena3modal.local_server --model-path $(pwd)/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000


Output:
Chunk ID: 7e5296
Wall time: 1.0013 seconds
Process running with session ID 49050
Original token count: 481
Output:
2026-02-17 00:58:57 | INFO     | veena3.local | Auth bypass enabled (local mode). Use --auth to require API keys.
2026-02-17 00:58:57 | INFO     | veena3.local | Resolved model paths:
2026-02-17 00:58:57 | INFO     | veena3.local |   LLM (vLLM):    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:58:57 | INFO     | veena3.local |   BiCodec:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:58:57 | INFO     | veena3.local | ============================================================
2026-02-17 00:58:57 | INFO     | veena3.local | Initializing Veena3 TTS Runtime (local)
2026-02-17 00:58:57 | INFO     | veena3.local |   Model:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:58:57 | INFO     | veena3.local |   LLM path:    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:58:57 | INFO     | veena3.local |   BiCodec:     /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:58:57 | INFO     | veena3.local |   Device:      cuda
2026-02-17 00:58:57 | INFO     | veena3.local |   GPU Memory:  10% (total)
2026-02-17 00:58:57 | INFO     | veena3.local |   Engines:     1
2026-02-17 00:58:57 | INFO     | veena3.local |   SR:          disabled
2026-02-17 00:58:57 | INFO     | veena3.local | ============================================================
2026-02-17 00:58:57 | INFO     | veena3modal.services.tts_runtime | Initializing TTS runtime...
2026-02-17 00:58:57 | INFO     | veena3modal.services.tts_runtime |   Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:58:57 | INFO     | veena3modal.services.tts_runtime |   BiCodec path: /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:58:57 | INFO     | veena3modal.services.tts_runtime |   BiCodec device: cpu
2026-02-17 00:58:57 | INFO     | veena3modal.services.tts_runtime |   SR enabled: False, path: None


I applied the BiCodec-device fallback and restarted the server with `BICODEC_DEVICE=cpu`; continuing init now to confirm startup and then I’ll launch the optimized stress script with concurrency sweeps.


Output:
Chunk ID: 6e1ef7
Wall time: 5.0010 seconds
Process running with session ID 49050
Original token count: 1295
Output:
2026-02-17 00:59:02 | INFO     | veena3modal.services.tts_runtime | Loading SparkTTS model with vLLM...
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.1
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
INFO 02-17 00:59:03 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 00:59:03 [model.py:1561] Using max model len 4096
INFO 02-17 00:59:03 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 00:59:03 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:04 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}


Output:
Chunk ID: c1455d
Wall time: 5.0011 seconds
Process running with session ID 49050
Original token count: 440
Output:
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:06 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:37857 backend=nccl
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:06 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:07 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:08 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=3723573) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=3723573) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=3723573)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=3723573)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.95it/s]
(EngineCore_DP0 pid=3723573)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.94it/s]
(EngineCore_DP0 pid=3723573)
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:08 [default_loader.py:291] Loading weights took 0.59 seconds
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:09 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.137132 seconds


Output:
Chunk ID: 096107
Wall time: 5.0009 seconds
Process running with session ID 49050
Original token count: 140
Output:
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:14 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:14 [backends.py:872] Dynamo bytecode transform time: 4.23 s
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:17 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.682 s
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:17 [monitor.py:34] torch.compile takes 4.91 s in total


Output:
Chunk ID: 00cd38
Wall time: 5.0012 seconds
Process running with session ID 49050
Original token count: 688
Output:
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:18 [gpu_worker.py:356] Available KV cache memory: 6.17 GiB
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:18 [kv_cache_utils.py:1307] GPU KV cache size: 538,928 tokens
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:18 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 131.57x
(EngineCore_DP0 pid=3723573)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%| | 0/35 [00:00<?,
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%| | 4/35 [00:00<00:
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|▏| 8/35 [00:00<00:
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|▎| 12/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|▍| 16/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|▌| 20/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|▋| 24/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|▊| 28/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|▉| 32/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|█| 35/35 [00:00<00
(EngineCore_DP0 pid=3723573)
Capturing CUDA graphs (decode, FULL):   0%|              | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|▉     | 3/19 [00:00<00:00, 29.82it/s]
Capturing CUDA graphs (decode, FULL):  37%|██▏   | 7/19 [00:00<00:00, 33.69it/s]
Capturing CUDA graphs (decode, FULL):  58%|██▉  | 11/19 [00:00<00:00, 34.86it/s]
Capturing CUDA graphs (decode, FULL):  79%|███▉ | 15/19 [00:00<00:00, 35.57it/s]
Capturing CUDA graphs (decode, FULL): 100%|█████| 19/19 [00:00<00:00, 36.37it/s]
Capturing CUDA graphs (decode, FULL): 100%|█████| 19/19 [00:00<00:00, 35.39it/s]
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:21 [gpu_model_runner.py:5063] Graph capturing finished in 3 secs, took 0.28 GiB
(EngineCore_DP0 pid=3723573) INFO 02-17 00:59:21 [core.py:272] init engine (profile, create kv cache, warmup model) took 11.31 seconds
✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

2026-02-17 00:59:22 | INFO     | veena3modal.services.tts_runtime | Initializing prompt builder...
2026-02-17 00:59:22 | INFO     | veena3modal.services.tts_runtime | Initializing BiCodec decoder...
🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cpu
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


Output:
Chunk ID: 9639ee
Wall time: 5.0003 seconds
Process running with session ID 49050
Original token count: 293
Output:
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
2026-02-17 00:59:28 | INFO     | veena3modal.services.tts_runtime | Initializing TTS pipeline...
🚀 SparkTTSPipeline initialized
2026-02-17 00:59:28 | INFO     | veena3modal.services.tts_runtime | Initializing streaming pipeline...
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)
2026-02-17 00:59:28 | INFO     | veena3modal.services.tts_runtime | ✅ TTS runtime initialized in 30852ms
2026-02-17 00:59:28 | INFO     | veena3modal.services.tts_runtime |    Model version: LLM
2026-02-17 00:59:28 | INFO     | veena3modal.services.tts_runtime | Pre-computing global tokens for all speakers...
2026-02-17 00:59:28 | INFO     | veena3modal.services.tts_runtime |   Cached globals for lipakshi
2026-02-17 00:59:29 | INFO     | veena3modal.services.tts_runtime |   Cached globals for vardan
2026-02-17 00:59:29 | INFO     | veena3modal.services.tts_runtime |   Cached globals for reet
2026-02-17 00:59:29 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nandini


Output:
Chunk ID: 88c519
Wall time: 5.0007 seconds
Process running with session ID 49050
Original token count: 696
Output:
2026-02-17 00:59:30 | INFO     | veena3modal.services.tts_runtime |   Cached globals for krishna
2026-02-17 00:59:30 | INFO     | veena3modal.services.tts_runtime |   Cached globals for anika
2026-02-17 00:59:31 | INFO     | veena3modal.services.tts_runtime |   Cached globals for adarsh
2026-02-17 00:59:31 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nilay
2026-02-17 00:59:31 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Aarvi
2026-02-17 00:59:32 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Asha
2026-02-17 00:59:32 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Bittu
2026-02-17 00:59:33 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Mira
2026-02-17 00:59:33 | INFO     | veena3modal.services.tts_runtime | Speaker globals cached: 12/12 speakers in 4863ms
2026-02-17 00:59:33 | INFO     | veena3.local | Runtime initialized in 35.7s
2026-02-17 00:59:33 | INFO     | veena3.local |   Model version: LLM
2026-02-17 00:59:33 | INFO     | veena3.local |   Streaming: available
2026-02-17 00:59:33 | INFO     | veena3.local |   Super-resolution: disabled
2026-02-17 00:59:33 | INFO     | veena3.local |
2026-02-17 00:59:33 | INFO     | veena3.local | ============================================================
2026-02-17 00:59:33 | INFO     | veena3.local | Veena3 TTS Local Server starting
2026-02-17 00:59:33 | INFO     | veena3.local |   URL: http://0.0.0.0:8000
2026-02-17 00:59:33 | INFO     | veena3.local |   Health: http://0.0.0.0:8000/v1/tts/health
2026-02-17 00:59:33 | INFO     | veena3.local |   Generate: POST http://0.0.0.0:8000/v1/tts/generate
2026-02-17 00:59:33 | INFO     | veena3.local |   Metrics: http://0.0.0.0:8000/v1/tts/metrics
2026-02-17 00:59:33 | INFO     | veena3.local |   Docs: http://0.0.0.0:8000/docs
2026-02-17 00:59:33 | INFO     | veena3.local | ============================================================
{"timestamp": "2026-02-17T00:59:33.101346+00:00", "level": "INFO", "logger": "veena3modal.api.websocket_handler", "message": "WebSocket route /v1/tts/ws registered"}
2026-02-17 00:59:33 | INFO     | veena3modal.api.websocket_handler | WebSocket route /v1/tts/ws registered
{"timestamp": "2026-02-17T00:59:33.101496+00:00", "level": "INFO", "logger": "veena3modal.api.fastapi_app", "message": "WebSocket support enabled at /v1/tts/ws"}
2026-02-17 00:59:33 | INFO     | veena3modal.api.fastapi_app | WebSocket support enabled at /v1/tts/ws
INFO:     Started server process [3723465]
INFO:     Waiting for application startup.
INFO:     Application startup complete.
INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)


Server is now healthy and ready. I’ll run the optimized stress script with 1,2,4 concurrent short requests first, capture JSON output, and then extend to higher concurrency if there are no immediate failures.


cd /home/ubuntu/veenaModal && python3 scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 256 --no-stream --chunking --output /tmp/bench_short_lvl_1_2_4.json


Output:
Chunk ID: ded4d6
Wall time: 0.0637 seconds
Process exited with code 2
Original token count: 105
Output:
usage: stress_test_optimized_local.py [-h] [--url URL] [--levels LEVELS]
                                      [--text {short,medium,long,mixed}]
                                      [--chunking] [--no-chunking]
                                      [--max-tokens MAX_TOKENS] [--stream]
                                      [--output OUTPUT]
stress_test_optimized_local.py: error: unrecognized arguments: --no-stream


cd /home/ubuntu/veenaModal && python3 scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 256 --chunking --output /tmp/bench_short_lvl_1_2_4.json


Output:
Chunk ID: b3ceac
Wall time: 1.0005 seconds
Process running with session ID 61231
Original token count: 0
Output:


Output:
Chunk ID: 93bc77
Wall time: 5.0011 seconds
Process running with session ID 61231
Original token count: 0
Output:


Output:
Chunk ID: ff3e48
Wall time: 5.0007 seconds
Process running with session ID 61231
Original token count: 0
Output:


Output:
Chunk ID: eb961e
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 689
Output:
  Health: healthy
  Version: LLM
  GPU: True

Warming up (1 request, non-streaming)...
  warmup ok: 0%

========================================================================
  1C-SHORT-NON-STREAMING
  10 requests, 1 concurrent, non-streaming, chunking=True
========================================================================
  Success:     0/10 (0%)
  Wall time:   2.42s
  Throughput:  4.13 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 81150MB avg, 81152MB peak / 81920MB
    Util:   3% avg, 4% peak
    Temp:   31C avg, 31C peak

  Errors:
    [10x] {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryA
  cooldown 3s...

========================================================================
  2C-SHORT-NON-STREAMING
  10 requests, 2 concurrent, non-streaming, chunking=True
========================================================================
  Success:     0/10 (0%)
  Wall time:   2.40s
  Throughput:  4.16 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 81151MB avg, 81151MB peak / 81920MB
    Util:   2% avg, 4% peak
    Temp:   31C avg, 31C peak

  Errors:
    [10x] {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryA
  cooldown 3s...

========================================================================
  4C-SHORT-NON-STREAMING
  12 requests, 4 concurrent, non-streaming, chunking=True
========================================================================
  Success:     0/12 (0%)
  Wall time:   2.84s
  Throughput:  4.22 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 81144MB avg, 81151MB peak / 81920MB
    Util:   4% avg, 5% peak
    Temp:   31C avg, 31C peak

  Errors:
    [12x] {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryA

==============================================================================================================
  STRESS BENCHMARK SUMMARY
==============================================================================================================
Level                        OK%     RPS     p50     p95  GPU%   GPUmem    RTF Inflight
--------------------------------------------------------------------------------------------------------------
1c-short-non-streaming       0%    4.13     ---     ---    3%   81152MB  0.000        0
2c-short-non-streaming       0%    4.16     ---     ---    2%   81151MB  0.000        0
4c-short-non-streaming       0%    4.22     ---     ---    4%   81151MB  0.000        0
==============================================================================================================

Results written to /tmp/bench_short_lvl_1_2_4.json


Output:
Chunk ID: 64de04
Wall time: 5.0029 seconds
Process running with session ID 49050
Original token count: 46102
Output:
Total output lines: 1222

INFO:     127.0.0.1:48076 - "GET /v1/tts/health HTTP/1.1" 200 OK
{"timestamp": "2026-02-17T00:59:40.083930+00:00", "level": "INFO", "logger": "veena3modal.tts", "message": "request_received", "request_id": "3f2eb95e-2c97-4488-ac28-7dfcd3e20000", "event": "request_received", "text_length": 28, "speaker": "lipakshi", "stream": false, "format": "wav"}
2026-02-17 00:59:40 | INFO     | veena3modal.tts | request_received
2026-02-17 00:59:40 | INFO     | veena3modal.services.tts_runtime | tts_runtime_chunking_start
2026-02-17 00:59:40 | INFO     | veena3modal.processing.long_text_processor | LongTextProcessor initialized
2026-02-17 00:59:40 | INFO     | veena3modal.core.pipeline | tts_llm_generate_start
2026-02-17 00:59:40 | ERROR    | veena3modal.services.tts_runtime | Chunked speech generation failed: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

{"timestamp": "2026-02-17T00:59:40.388568+00:00", "level": "ERROR", "logger": "veena3modal.api.fastapi_app", "message": "TTS generation error: CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n", "request_id": "3f2eb95e-2c97-4488-ac28-7dfcd3e20000", "exception": "Traceback (most recent call last):\n  File \"/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py\", line 446, in tts_generate\n    audio_bytes, metrics = await tts_runtime.generate_speech_chunked(\n  File \"/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\", line 792, in generate_speech_chunked\n    audio_bytes, pipeline_timings = await generation_coro\n  File \"/home/ubuntu/veenaModal/veena3modal/core/pipeline.py\", line 401, in generate_speech_indic\n    return await self.generate_speech(\n  File \"/home/ubuntu/veenaModal/veena3modal/core/pipeline.py\", line 165, in generate_speech\n    torch.cuda.synchronize()\n  File \"/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py\", line 1083, in synchronize\n    return torch._C._cuda_synchronize()\ntorch.AcceleratorError: CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n\n"}
2026-02-17 00:59:40 | ERROR    | veena3modal.api.fastapi_app | TTS generation error: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py", line 446, in tts_generate
    audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 792, in generate_speech_chunked
    audio_bytes, pipeline_timings = await generation_coro
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 401, in generate_speech_indic
    return await self.generate_speech(
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 165, in generate_speech
    torch.cuda.synchronize()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1083, in synchronize
    return torch._C._cuda_synchronize()
torch.AcceleratorError: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

{"timestamp": "2026-02-17T00:59:40.389603+00:00", "level": "ERROR", "logger": "veena3modal.tts", "message": "request_failed", "request_id": "3f2eb95e-2c97-4488-ac28-7dfcd3e20000", "event": "request_failed", "status_code": 500, "error_code": "INTERNAL_ERROR", "error_message": "CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n"}
2026-02-17 00:59:40 | ERROR    | veena3modal.tts | request_failed
INFO:     127.0.0.1:48082 - "POST /v1/tts/generate HTTP/1.1" 500 Internal Server Error
{"timestamp": "2026-02-17T00:59:40.441934+00:00", "level": "INFO", "logger": "veena3modal.tts", "message": "request_received", "request_id": "0c31ebd0-a55f-4c5f-a33b-60d0a30a165e", "event": "request_received", "text_length": 28, "speaker": "lipakshi", "stream": false, "format": "wav"}
2026-02-17 00:59:40 | INFO     | veena3modal.tts | request_received
2026-02-17 00:59:40 | INFO     | veena3modal.services.tts_runtime | tts_runtime_chunking_start
2026-02-17 00:59:40 | INFO     | veena3modal.processing.long_text_processor | LongTextProcessor initialized
2026-02-17 00:59:40 | INFO     | veena3modal.core.pipeline | tts_llm_generate_start
2026-02-17 00:59:40 | ERROR    | veena3modal.services.tts_runtime | Chunked speech generation failed: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

{"timestamp": "2026-02-17T00:59:40.682561+00:00", "level": "ERROR", "logger": "veena3modal.api.fastapi_app", "message": "TTS generation error: CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n", "request_id": "0c31ebd0-a55f-4c5f-a33b-60d0a30a165e", "exception": "Traceback (most recent call last):\n  File \"/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py\", line 446, in tts_generate\n    audio_bytes, metrics = await tts_runtime.generate_speech_chunked(\n  File \"/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\", line 792, in generate_speech_chunked\n    audio_bytes, pipeline_timings = await generation_coro\n  File \"/home/ubuntu/veenaModal/veena3modal/core/pipeline.py\", line 401, in generate_speech_indic\n    return await self.generate_speech(\n  File \"/home/ubuntu/veenaM…42102 tokens truncated…docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py", line 446, in tts_generate
    audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 792, in generate_speech_chunked
    audio_bytes, pipeline_timings = await generation_coro
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 401, in generate_speech_indic
    return await self.generate_speech(
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 165, in generate_speech
    torch.cuda.synchronize()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1083, in synchronize
    return torch._C._cuda_synchronize()
torch.AcceleratorError: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

{"timestamp": "2026-02-17T00:59:53.825999+00:00", "level": "ERROR", "logger": "veena3modal.tts", "message": "request_failed", "request_id": "11286b43-d0e9-4381-8002-60590b567020", "event": "request_failed", "status_code": 500, "error_code": "INTERNAL_ERROR", "error_message": "CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n"}
2026-02-17 00:59:53 | ERROR    | veena3modal.tts | request_failed
INFO:     127.0.0.1:39998 - "POST /v1/tts/generate HTTP/1.1" 500 Internal Server Error
{"timestamp": "2026-02-17T00:59:53.827437+00:00", "level": "INFO", "logger": "veena3modal.tts", "message": "request_received", "request_id": "680ce77b-66bd-41c3-b661-ba983ebf95b5", "event": "request_received", "text_length": 28, "speaker": "anika", "stream": false, "format": "wav"}
2026-02-17 00:59:53 | INFO     | veena3modal.tts | request_received
2026-02-17 00:59:53 | INFO     | veena3modal.services.tts_runtime | tts_runtime_chunking_start
2026-02-17 00:59:53 | INFO     | veena3modal.processing.long_text_processor | LongTextProcessor initialized
2026-02-17 00:59:53 | INFO     | veena3modal.core.pipeline | tts_llm_generate_start
2026-02-17 00:59:54 | ERROR    | veena3modal.services.tts_runtime | Chunked speech generation failed: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

{"timestamp": "2026-02-17T00:59:54.057909+00:00", "level": "ERROR", "logger": "veena3modal.api.fastapi_app", "message": "TTS generation error: CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n", "request_id": "680ce77b-66bd-41c3-b661-ba983ebf95b5", "exception": "Traceback (most recent call last):\n  File \"/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py\", line 446, in tts_generate\n    audio_bytes, metrics = await tts_runtime.generate_speech_chunked(\n  File \"/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\", line 792, in generate_speech_chunked\n    audio_bytes, pipeline_timings = await generation_coro\n  File \"/home/ubuntu/veenaModal/veena3modal/core/pipeline.py\", line 401, in generate_speech_indic\n    return await self.generate_speech(\n  File \"/home/ubuntu/veenaModal/veena3modal/core/pipeline.py\", line 165, in generate_speech\n    torch.cuda.synchronize()\n  File \"/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py\", line 1083, in synchronize\n    return torch._C._cuda_synchronize()\ntorch.AcceleratorError: CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n\n"}
2026-02-17 00:59:54 | ERROR    | veena3modal.api.fastapi_app | TTS generation error: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py", line 446, in tts_generate
    audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 792, in generate_speech_chunked
    audio_bytes, pipeline_timings = await generation_coro
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 401, in generate_speech_indic
    return await self.generate_speech(
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 165, in generate_speech
    torch.cuda.synchronize()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py", line 1083, in synchronize
    return torch._C._cuda_synchronize()
torch.AcceleratorError: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

{"timestamp": "2026-02-17T00:59:54.058369+00:00", "level": "ERROR", "logger": "veena3modal.tts", "message": "request_failed", "request_id": "680ce77b-66bd-41c3-b661-ba983ebf95b5", "event": "request_failed", "status_code": 500, "error_code": "INTERNAL_ERROR", "error_message": "CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n"}
2026-02-17 00:59:54 | ERROR    | veena3modal.tts | request_failed
INFO:     127.0.0.1:40012 - "POST /v1/tts/generate HTTP/1.1" 500 Internal Server Error


The benchmark run exposed a persistent CUDA OOM at request entry (`torch.cuda.synchronize()` in `pipeline.generate_speech`), so no requests completed. I’ll inspect the JSON output and captured logs to confirm whether it’s kernel/first-batch initialization vs sustained memory pressure.


python3 - <<'PY'
import json
from pprint import pprint
p='/tmp/bench_short_lvl_1_2_4.json'
with open(p) as f:
    data=json.load(f)
print('levels:', len(data))
for level in data:
    print('\n', level['level'], level['concurrency'], 'success', level['success_rate'], 'errors', level['errors'])
    # print first few requests
    for req in level['requests'][:3]:
        print(' req', req['request_id'], req['status_code'], req.get('error'))
PY


Output:
Chunk ID: e93a26
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 676
Output:
levels: 3

 1c-short-non-streaming 1 success 0.0 errors {'{"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\\nSearch for `cudaErrorMemoryA': 10}
 req 0c31ebd0-a55f-4c5f-a33b-60d0a30a165e 500 {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more infor
 req 2a5d906f-60e5-4d11-8e55-ed877a619b49 500 {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more infor
 req c5642a0e-a018-4541-8b31-7caa7d518fdb 500 {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more infor

 2c-short-non-streaming 2 success 0.0 errors {'{"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\\nSearch for `cudaErrorMemoryA': 10}
 req c6b8bd23-24dd-40aa-ab70-a2dd06f740fd 500 {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more infor
 req 02fab234-e8f7-4b87-b553-9bf6e9fb8747 500 {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more infor
 req cfc9bac0-ccc1-4d05-b041-5a859913b93d 500 {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more infor

 4c-short-non-streaming 4 success 0.0 errors {'{"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\\nSearch for `cudaErrorMemoryA': 12}
 req fd2b69cc-b075-4e2c-9b65-4175841f4603 500 {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more infor
 req 5b2ff47d-ee71-4bfe-a772-720b5e678b64 500 {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more infor
 req b7f8903a-4f2a-4a9c-99ae-d8244dbe1dcb 500 {"error":{"code":"INTERNAL_ERROR","message":"CUDA error: out of memory\nSearch for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more infor


cd /home/ubuntu/veenaModal && VEENA3_TIMING_LOGS=true NUMBA_DISABLE_JIT=1 BICODEC_DEVICE=cpu GPU_MEMORY_UTILIZATION=0.1 python3 - <<'PY'
import asyncio, os
from veena3modal.services import tts_runtime

async def main():
    tts_runtime.initialize_runtime(
        model_path='models/spark_tts_4speaker/LLM',
        bicodec_path='models/spark_tts_4speaker',
        device='cuda',
        gpu_memory_utilization=0.1,
        num_engines=1,
        bicodec_device='cpu',
    )
    audio, metrics = await tts_runtime.generate_speech(
        text='Hello, this is a quick test.',
        speaker='lipakshi',
        max_tokens=256,
    )
    print('len', len(audio) if audio else None)
    print('metrics', {k:v for k,v in metrics.items() if k in ['llm_token_total','llm_batch_count','generation_ms','audio_duration_seconds','request_id']})

asyncio.run(main())
PY


Output:
Chunk ID: 800b2f
Wall time: 1.0015 seconds
Process running with session ID 47233
Original token count: 0
Output:


Output:
Chunk ID: 425a71
Wall time: 5.0013 seconds
Process running with session ID 47233
Original token count: 1253
Output:
[2026-02-17 01:00:12] INFO tts_runtime.py:192: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.1
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-17 01:00:13 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 01:00:13 [model.py:1561] Using max model len 4096
INFO 02-17 01:00:13 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 01:00:13 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3724187) INFO 02-17 01:00:14 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}


Output:
Chunk ID: 8c049a
Wall time: 2.7785 seconds
Process exited with code 1
Original token count: 2668
Output:
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946] EngineCore failed to start.
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946] Traceback (most recent call last):
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 937, in run_engine_core
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]     engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 691, in __init__
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]     super().__init__(
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 105, in __init__
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]     self.model_executor = executor_class(vllm_config)
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/executor/abstract.py", line 101, in __init__
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]     self._init_executor()
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/executor/uniproc_executor.py", line 47, in _init_executor
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]     self.driver_worker.init_device()
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]     self.worker.init_device()  # type: ignore
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]     current_platform.set_device(self.device)
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/platforms/cuda.py", line 123, in set_device
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]     torch.cuda.set_device(device)
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py", line 567, in set_device
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]     torch._C._cuda_setDevice(device)
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946] torch.AcceleratorError: CUDA error: out of memory
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946] Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946] CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946] For debugging consider passing CUDA_LAUNCH_BLOCKING=1
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946] Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
(EngineCore_DP0 pid=3724187) ERROR 02-17 01:00:16 [core.py:946]
(EngineCore_DP0 pid=3724187) Process EngineCore_DP0:
(EngineCore_DP0 pid=3724187) Traceback (most recent call last):
(EngineCore_DP0 pid=3724187)   File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
(EngineCore_DP0 pid=3724187)     self.run()
(EngineCore_DP0 pid=3724187)   File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
(EngineCore_DP0 pid=3724187)     self._target(*self._args, **self._kwargs)
(EngineCore_DP0 pid=3724187)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 950, in run_engine_core
(EngineCore_DP0 pid=3724187)     raise e
(EngineCore_DP0 pid=3724187)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 937, in run_engine_core
(EngineCore_DP0 pid=3724187)     engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)
(EngineCore_DP0 pid=3724187)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 691, in __init__
(EngineCore_DP0 pid=3724187)     super().__init__(
(EngineCore_DP0 pid=3724187)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 105, in __init__
(EngineCore_DP0 pid=3724187)     self.model_executor = executor_class(vllm_config)
(EngineCore_DP0 pid=3724187)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/executor/abstract.py", line 101, in __init__
(EngineCore_DP0 pid=3724187)     self._init_executor()
(EngineCore_DP0 pid=3724187)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/executor/uniproc_executor.py", line 47, in _init_executor
(EngineCore_DP0 pid=3724187)     self.driver_worker.init_device()
(EngineCore_DP0 pid=3724187)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device
(EngineCore_DP0 pid=3724187)     self.worker.init_device()  # type: ignore
(EngineCore_DP0 pid=3724187)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device
(EngineCore_DP0 pid=3724187)     current_platform.set_device(self.device)
(EngineCore_DP0 pid=3724187)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/platforms/cuda.py", line 123, in set_device
(EngineCore_DP0 pid=3724187)     torch.cuda.set_device(device)
(EngineCore_DP0 pid=3724187)   File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py", line 567, in set_device
(EngineCore_DP0 pid=3724187)     torch._C._cuda_setDevice(device)
(EngineCore_DP0 pid=3724187) torch.AcceleratorError: CUDA error: out of memory
(EngineCore_DP0 pid=3724187) Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
(EngineCore_DP0 pid=3724187) CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
(EngineCore_DP0 pid=3724187) For debugging consider passing CUDA_LAUNCH_BLOCKING=1
(EngineCore_DP0 pid=3724187) Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
(EngineCore_DP0 pid=3724187)
[2026-02-17 01:00:16] ERROR tts_runtime.py:295: ❌ Failed to initialize TTS runtime: Engine core initialization failed. See root cause above. Failed core proc(s): {}
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 193, in initialize_runtime
    model = SparkTTSModel(
  File "/home/ubuntu/veenaModal/veena3modal/core/model_loader.py", line 128, in __init__
    self.engine = AsyncLLMEngine.from_engine_args(engine_args)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 257, in from_engine_args
    return cls(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 155, in __init__
    self.engine_core = EngineCoreClient.make_async_mp_client(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 122, in make_async_mp_client
    return AsyncMPClient(*client_args)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 819, in __init__
    super().__init__(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 479, in __init__
    with launch_core_engines(vllm_config, executor_class, log_stats) as (
  File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__
    next(self.gen)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/utils.py", line 933, in launch_core_engines
    wait_for_engine_startup(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/utils.py", line 992, in wait_for_engine_startup
    raise RuntimeError(
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 193, in initialize_runtime
    model = SparkTTSModel(
  File "/home/ubuntu/veenaModal/veena3modal/core/model_loader.py", line 128, in __init__
    self.engine = AsyncLLMEngine.from_engine_args(engine_args)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 257, in from_engine_args
    return cls(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 155, in __init__
    self.engine_core = EngineCoreClient.make_async_mp_client(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 122, in make_async_mp_client
    return AsyncMPClient(*client_args)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 819, in __init__
    super().__init__(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 479, in __init__
    with launch_core_engines(vllm_config, executor_class, log_stats) as (
  File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__
    next(self.gen)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/utils.py", line 933, in launch_core_engines
    wait_for_engine_startup(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/utils.py", line 992, in wait_for_engine_startup
    raise RuntimeError(
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "<stdin>", line 21, in <module>
  File "/usr/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/usr/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "<stdin>", line 5, in main
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 298, in initialize_runtime
    raise RuntimeError(f"TTS runtime initialization failed: {e}") from e
RuntimeError: TTS runtime initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {}


Output:
Chunk ID: 12ec2d
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 67
Output:
3706516, VLLM::EngineCore, 13074
3708772, VLLM::EngineCore, 13074
3708601, python3, 2238
3713837, VLLM::EngineCore, 13074
3716918, VLLM::EngineCore, 13074
3716775, python3, 2238
3720983, VLLM::EngineCore, 13074
3720853, python3, 2238
3723573, VLLM::EngineCore, 8994


I found the actual reason for all request OOMs: multiple stale `VLLM::EngineCore` workers from earlier failed starts still live and consume ~13GB each, saturating VRAM. I’ll clean up the orphaned processes and re-run with a single fresh server before any timing pass.


Output:
Chunk ID: 57ff8d
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 222
Output:
UID          PID    PPID  C STIME TTY      STAT   TIME CMD
ubuntu   3706516       1  2 00:25 ?        Sl     0:48 VLLM::EngineCore
ubuntu   3708601 3708597  4 00:28 ?        Sl     1:20 python3 -
ubuntu   3708772 3708601  1 00:28 ?        Sl     0:36 VLLM::EngineCore
ubuntu   3713837       1  3 00:36 ?        Sl     0:50 VLLM::EngineCore
ubuntu   3716775 3716771  7 00:44 ?        Sl     1:14 python3 -
ubuntu   3716918 3716775  4 00:44 ?        Sl     0:43 VLLM::EngineCore
ubuntu   3720853 3720849 26 00:53 ?        Sl     1:43 python3 -
ubuntu   3720983 3720853  7 00:53 ?        Sl     0:28 VLLM::EngineCore
ubuntu   3723465 3675569 92 00:58 pts/9    Ssl+   1:21 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000
ubuntu   3723573 3723465 37 00:59 pts/9    Sl+    0:30 VLLM::EngineCore


for pid in 3706516 3708772 3708601 3713837 3716918 3716775 3720983 3720853; do kill -9 $pid 2>/dev/null || true; done; sleep 1; nvidia-smi --query-compute-apps=pid,name,used_memory --format=csv,noheader,nounits


Output:
Chunk ID: 610183
Wall time: 0.9277 seconds
Process exited with code 0
Original token count: 8
Output:
3723573, VLLM::EngineCore, 8994


VRAM is now clear except the active server’s engine (~9GB). I’ll rerun the stress script at low concurrency first to confirm successful generation, then scale up only if this first tier succeeds.


cd /home/ubuntu/veenaModal && python3 scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 128 --chunking --output /tmp/bench_short_lvl_1_2_4_v2.json


Output:
Chunk ID: c127b4
Wall time: 1.0013 seconds
Process running with session ID 29822
Original token count: 0
Output:


Output:
Chunk ID: ecab0e
Wall time: 5.0010 seconds
Process running with session ID 29822
Original token count: 0
Output:


The current TTS application here is an implementation in Modal labs. Which has autoscaling and other rules self scaling rules and everything suitable for my target audience. Now, i need you to create a local entry point or implementation point. then we will plan for optimising the pipeline, then we will discuss on stress testing, then we will discuss on scaling laws post optimisations locally. @veena3modal/app.py

the above part is performed by a different agent. and benchmarks are made. Now can you do the benchmarks on this newly optimized path to understand the concurrency and stress testing.

Now i need you to capture detailed time logs for each request happening in the GPU. like what happens with 1req, time per LLM token, time per batch decoding, and everything else. detailed time about where we are spending each minute in the GPU per generation.

and when you have the details for per request. you can obviously add more logging to get details of concurrency. so you understand what's the bottleneck for the peak concurrency. and then we will decide on effective methods in optimising this TTS engine.

<permissions instructions>
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `danger-full-access`: No filesystem sandboxing - all commands are permitted. Network access is enabled.
Approval policy is currently never. Do not provide the `sandbox_permissions` for any reason, commands will be rejected.
</permissions instructions>

<collaboration_mode># Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.
</collaboration_mode>

# AGENTS.md instructions for /home/ubuntu/veenaModal

<INSTRUCTIONS>
## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.
</INSTRUCTIONS>

<environment_context>
  <cwd>/home/ubuntu/veenaModal</cwd>
  <shell>bash</shell>
</environment_context>


Summary: auto


I’ve already got this in a runnable state with the local server and local stress script available. Next I’ll quickly verify the current benchmark script’s logging fields and run the benchmark sweep with detailed per-request timing extraction; if needed, I’ll add one focused logging enhancement in that script and re-run once.


sed -n '1,260p' scripts/stress_test_optimized_local.py


Output:
Chunk ID: 53d07a
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 2113
Output:
#!/usr/bin/env python3
"""Detailed local stress benchmark for optimized TTS path.

This script is focused on collecting per-request GPU/LLM timing for local tuning:
- Request wall-clock latency and server reported TTFB
- GPU timing breakdown (LLM batch wall / GPU wall, parse, decode)
- Time per token and per batch
- Concurrency visibility from in-flight counters
- End-to-end throughput and RTF with GPU utilization snapshots

Usage:
    python scripts/stress_test_optimized_local.py
    python scripts/stress_test_optimized_local.py --levels 1,5,10,20 --concurrency-scaling both
    python scripts/stress_test_optimized_local.py --text-category long --output benchmarks.json
"""

import argparse
import asyncio
import json
import statistics
import subprocess
import sys
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

import httpx

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

TEST_TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This is a baseline quality sentence used for benchmarking local TTS latency, "
        "TTFB, and tokenization behavior under load."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast shadows over crowded streets, "
        "there lived a quiet linguist who believed every sentence carried a rhythm. "
        "She recorded conversations at dawn and dusk, trying to capture how meaning shifted with breath. "
        "By evening she would review them, matching sound to emotion, and every night she learned a new way to speak."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    """Background polling for quick GPU health snapshots."""

    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=3)

    def _poll_loop(self):
        while self._running:
            try:
                result = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True,
                    text=True,
                    timeout=5,
                )
                if result.returncode == 0 and result.stdout:
                    parts = [p.strip() for p in result.stdout.strip().split(",")]
                    if len(parts) >= 4:
                        self.snapshots.append(
                            GPUSnapshot(
                                timestamp=time.time(),
                                memory_used_mb=float(parts[0]),
                                memory_total_mb=float(parts[1]),
                                gpu_utilization_pct=float(parts[2]),
                                temperature_c=float(parts[3]),
                            )
                        )
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {
                "samples": 0,
                "error": "no GPU samples captured",
            }

        mem_used = [s.memory_used_mb for s in self.snapshots]
        gpu_util = [s.gpu_utilization_pct for s in self.snapshots]
        temp = [s.temperature_c for s in self.snapshots]

        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem_used),
                "max": max(mem_used),
                "avg": statistics.mean(mem_used),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(gpu_util),
                "max": max(gpu_util),
                "avg": statistics.mean(gpu_util),
            },
            "temperature_c": {
                "min": min(temp),
                "max": max(temp),
                "avg": statistics.mean(temp),
            },
        }

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float
    ttfb_ms: float = 0.0
    server_ttfb_ms: float = 0.0
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    request_id: str = ""
    error: Optional[str] = None
    speaker: str = ""
    text_length: int = 0
    stream: bool = False
    timing: Dict[str, Any] = field(default_factory=dict)
    request_inflight: Optional[int] = None

@dataclass
class LevelResult:
    level_name: str
    concurrency: int
    total_requests: int
    mode: str
    text_category: str
    chunking: bool
    results: List[RequestResult] = field(default_factory=list)
    wall_time_seconds: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0.0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_seconds if self.wall_time_seconds > 0 else 0.0

    @property
    def audio_duration_total(self) -> float:
        return sum(r.audio_seconds for r in self.successes)

    @property
    def effective_rtf(self) -> float:
        total_audio = self.audio_duration_total
        return self.wall_time_seconds / total_audio if total_audio > 0 else 0.0

    def _percentile(self, values: List[float], pct: float) -> float:
        if not values:
            return 0.0
        sorted_vals = sorted(values)
        idx = int(len(sorted_vals) * pct / 100)
        return sorted_vals[min(idx, len(sorted_vals) - 1)]

    @property
    def latency_stats(self) -> Dict[str, float]:
        lat = [r.latency_ms for r in self.successes]
        if not lat:
            return {}
        return {
            "min": min(lat),
            "avg": statistics.mean(lat),
            "p50": self._percentile(lat, 50),
            "p95": self._percentile(lat, 95),
            "p99": self._percentile(lat, 99),
            "max": max(lat),
        }

    @property
    def ttfb_stats(self) -> Dict[str, float]:
        vals = [r.server_ttfb_ms for r in self.successes if r.server_ttfb_ms >= 0]
        if not vals:
            return {}
        return {
            "min": min(vals),
            "avg": statistics.mean(vals),
            "p50": self._percentile(vals, 50),
            "p95": self._percentile(vals, 95),
            "max": max(vals),
        }

    def timing_stats(self, key: str) -> Dict[str, float]:
        values: List[float] = []
        for r in self.successes:
            v = r.timing.get(key)
            if isinstance(v, (int, float)):
                values.append(float(v))
        if not values:
            return {}
        return {
            "min": min(values),
            "avg": statistics.mean(values),
            "p50": self._percentile(values, 50),
            "p95": self._percentile(values, 95),
            "p99": self._percentile(values, 99),
            "max": max(values),
        }

    def timing_rate(self, numerator_key: str, denominator_key: str) -> Tuple[float, int]:
        num = 0.0
        den = 0
        for r in self.successes:
            n = r.timing.get(numerator_key)
            d = r.timing.get(denominator_key)
            if isinstance(n, (int, float)) and isinstance(d, (int, float)) and d > 0:
                num += float(n)
                den += int(d)
        if den == 0:
            return 0.0, 0
        return num / den, den


sed -n '260,560p' scripts/stress_test_optimized_local.py