Output:
Chunk ID: bb0004
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 3153
Output:
#!/usr/bin/env python3
"""
Local Stress Test for Veena3 TTS Server.

Progressive load ramp with GPU monitoring, latency percentiles,
error categorization, and throughput measurement.

Unlike the Modal load tests, this runs locally with:
- No network latency (localhost)
- Direct GPU monitoring (nvidia-smi + torch.cuda)
- Higher concurrency ceiling (single-machine, no rate limits)
- Real streaming TTFB measurement (byte-level timing)

Usage:
    # Full suite (ramps 1 → 5 → 10 → 20 → 50 → 100 concurrent)
    python scripts/stress_test_local.py

    # Quick smoke test (1 → 5 → 10 only)
    python scripts/stress_test_local.py --quick

    # Custom ramp
    python scripts/stress_test_local.py --levels 1,5,10,20

    # Streaming only
    python scripts/stress_test_local.py --stream-only

    # Non-streaming only
    python scripts/stress_test_local.py --no-stream

    # Custom endpoint
    python scripts/stress_test_local.py --url http://localhost:8080
"""

import argparse
import asyncio
import json
import os
import statistics
import subprocess
import sys
import time
import threading
from dataclasses import dataclass, field
from typing import List, Optional, Dict, Any, Tuple

import httpx

# === Configuration ===

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

# Test texts at different lengths to stress different parts of the pipeline
TEST_TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This sentence tests basic text normalization and generation quality "
        "with moderate length input that should complete in reasonable time."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast long shadows "
        "over crowded streets, there lived an old bookkeeper named Margaret. She had spent "
        "forty years cataloging the stories of others, yet never found time to write her own. "
        "One rainy Tuesday afternoon, she discovered a leather-bound journal tucked behind "
        "a shelf of dusty encyclopedias. Its pages were blank, waiting. She picked up her pen "
        "and began to write. The words flowed like water from a broken dam."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

# === GPU Monitoring ===

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    """Background thread that polls nvidia-smi for GPU stats."""

    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=5)

    def _poll_loop(self):
        while self._running:
            try:
                result = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True, text=True, timeout=5,
                )
                if result.returncode == 0:
                    parts = result.stdout.strip().split(", ")
                    if len(parts) >= 4:
                        self.snapshots.append(GPUSnapshot(
                            timestamp=time.time(),
                            memory_used_mb=float(parts[0]),
                            memory_total_mb=float(parts[1]),
                            gpu_utilization_pct=float(parts[2]),
                            temperature_c=float(parts[3]),
                        ))
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {"error": "no GPU data collected"}

        mem_used = [s.memory_used_mb for s in self.snapshots]
        gpu_util = [s.gpu_utilization_pct for s in self.snapshots]
        temps = [s.temperature_c for s in self.snapshots]

        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem_used),
                "max": max(mem_used),
                "avg": statistics.mean(mem_used),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(gpu_util),
                "max": max(gpu_util),
                "avg": statistics.mean(gpu_util),
            },
            "temperature_c": {
                "min": min(temps),
                "max": max(temps),
                "avg": statistics.mean(temps),
            },
        }

# === Request Result ===

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float  # Total wall-clock time
    ttfb_ms: float = 0.0  # Time to first byte (from HTTP response start)
    server_ttfb_ms: float = 0.0  # Server-reported TTFB (X-TTFB-ms header)
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    error: Optional[str] = None
    speaker: str = ""
    text_length: int = 0
    stream: bool = False

# === Load Level Result ===

@dataclass
class LevelResult:
    level_name: str
    concurrency: int
    total_requests: int
    mode: str  # "non-streaming" or "streaming"
    text_category: str  # "short", "medium", "long", "mixed"
    results: List[RequestResult] = field(default_factory=list)
    wall_time_seconds: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_seconds if self.wall_time_seconds > 0 else 0

    def _percentile(self, values: List[float], pct: float) -> float:
        if not values:
            return 0.0
        s = sorted(values)
        idx = int(len(s) * pct / 100)
        return s[min(idx, len(s) - 1)]

    @property
    def latency_stats(self) -> Dict[str, float]:
        lats = [r.latency_ms for r in self.successes]
        if not lats:
            return {}
        return {
            "min": min(lats),
            "avg": statistics.mean(lats),
            "p50": self._percentile(lats, 50),
            "p95": self._percentile(lats, 95),
            "p99": self._percentile(lats, 99),
            "max": max(lats),
        }

    @property
    def ttfb_stats(self) -> Dict[str, float]:
        ttfbs = [r.ttfb_ms for r in self.successes if r.ttfb_ms > 0]
        if not ttfbs:
            return {}
        return {
            "min": min(ttfbs),
            "avg": statistics.mean(ttfbs),
            "p50": self._percentile(ttfbs, 50),
            "p95": self._percentile(ttfbs, 95),
            "max": max(ttfbs),
        }

    @property
    def audio_duration_total(self) -> float:
        return sum(r.audio_seconds for r in self.successes)

    @property
    def effective_rtf(self) -> float:
        """Wall time / total audio produced. Lower = better."""
        total_audio = self.audio_duration_total
        return self.wall_time_seconds / total_audio if total_audio > 0 else 0

    def error_summary(self) -> Dict[str, int]:
        errors: Dict[str, int] = {}
        for r in self.failures:
            key = r.error or f"HTTP {r.status_code}"
            errors[key[:80]] = errors.get(key[:80], 0) + 1
        return errors

# === HTTP Client ===

async def make_request(
    client: httpx.AsyncClient,
    base_url: str,
    text: str,
    speaker: str,
    stream: bool = False,
    timeout: float = 120.0,
) -> RequestResult:
    """Make a single TTS request with timing."""
    url = f"{base_url}{GENERATE_PATH}"
    payload = {
        "text": text,
        "speaker": speaker,
        "stream": stream,
        "format": "wav",
    }

    start = time.time()
    ttfb_time = 0.0

    try:
        if stream:
            # Streaming: measure true TTFB (time to first byte of audio)
            async with client.stream(
                "POST", url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            ) as response:
                first_byte = True
                total_bytes = 0
                async for chunk in response.aiter_bytes():
                    if first_byte:
                        ttfb_time = (time.time() - start) * 1000
                        first_byte = False
                    total_bytes += len(chunk)

                latency = (time.time() - start) * 1000

                if response.status_code == 200:
                    # Estimate audio duration from bytes (16kHz, 16-bit, mono, minus WAV header)
                    pcm_bytes = max(0, total_bytes - 44)
                    audio_seconds = pcm_bytes / (16000 * 2)

                    return RequestResult(
                        success=True,
                        status_code=200,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        audio_bytes=total_bytes,
                        audio_seconds=audio_seconds,
                        speaker=speaker,
                        text_length=len(text),
                        stream=True,
                    )
                else:
                    return RequestResult(
                        success=False,
                        status_code=response.status_code,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        error=f"HTTP {response.status_code}",
                        speaker=speaker,
                        text_length=len(text),
                        stream=True,
                    )
        else:
            # Non-streaming: standard request
            response = await client.post(
                url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            )
            latency = (time.time() - start) * 1000

            # Server-reported TTFB
            server_ttfb = 0.0
            ttfb_header = response.headers.get("X-TTFB-ms")
            if ttfb_header:
                try:
                    server_ttfb = float(ttfb_header)
                except ValueError:
                    pass

            # Audio duration from header or estimate
            audio_seconds = 0.0
            audio_sec_header = response.headers.get("X-Audio-Seconds")
            if audio_sec_header:
                try:
                    audio_seconds = float(audio_sec_header)
                except ValueError:
                    pass
            elif response.status_code == 200:
                pcm_bytes = max(0, len(response.content) - 44)
                audio_seconds = pcm_bytes / (16000 * 2)

            if response.status_code == 200:
                return RequestResult(
                    success=True,
                    status_code=200,
                    latency_ms=latency,
                    ttfb_ms=latency,  # Non-streaming: TTFB = total latency
                    server_ttfb_ms=server_ttfb,
                    audio_bytes=len(response.content),
                    audio_seconds=audio_seconds,
                    speaker=speaker,
                    text_length=len(text),
                    stream=False,
                )
            else:
                error_text = response.text[:200] if response.text else f"HTTP {response.status_code}"
                return RequestResult(
                    success=False,
                    status_code=response.status_code,
                    latency_ms=latency,
                    error=error_text,
                    speaker=speaker,
                    text_length=len(text),
                    stream=False,
                )

    except httpx.TimeoutException:


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


rg -n "def generate_speech_indic|class SparkTTSPipeline|generate_speech\(" veena3modal/core/pipeline.py


Output:
Chunk ID: d9c84a
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 78
Output:
52:class SparkTTSPipeline:
79:    async def generate_speech(
363:    async def generate_speech_indic(
380:        It delegates to generate_speech() with the same implementation.
400:        # Delegate to generate_speech() with the same implementation for Spark TTS
401:        return await self.generate_speech(


Summary: auto


sed -n '340,470p' veena3modal/core/pipeline.py


Output:
Chunk ID: d214e9
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 1035
Output:
            "llm_gpu_batch_ms_total": int(sum(batch_gpu_ms)),
            "llm_gpu_batch_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_p50": int(median(batch_gpu_ms)) if batch_gpu_ms else 0,
            "decode_cpu_ms": int(decode_ms),
            "parse_ms": int(parse_ms),
            "total_ms": int(total_ms),
        }
        if token_batches:
            timing_metrics["tokens_per_batch_min"] = min(token_batches)
            timing_metrics["tokens_per_batch_max"] = max(token_batches)
            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))

        if emit_timing:
            logger.info("tts_llm_complete", extra=timing_metrics)

        from veena3modal.audio.utils import add_wav_header
        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)

        if return_timing:
            return wav_bytes, timing_metrics
        return wav_bytes

    async def generate_speech_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ):
        """
        Generate speech audio for Spark TTS (speaker-based).

        This method is kept for backward compatibility with existing API.
        It delegates to generate_speech() with the same implementation.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "Hello <laugh> this is fun!" (will be normalized)
                - "नमस्ते! [excited] आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max BiCodec tokens to generate
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Delegate to generate_speech() with the same implementation for Spark TTS
        return await self.generate_speech(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
            return_timing=return_timing,
            request_id=request_id,
        )

    def _extract_bicodec_tokens(self, generated_text: str) -> Tuple[List[int], List[int]]:
        """
        Extract BiCodec semantic and global tokens from generated text using regex.

        Spark TTS generates tokens in the format:
        - <|bicodec_semantic_{id}|>
        - <|bicodec_global_{id}|>

        Args:
            generated_text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
            Returns ([], []) if no tokens found
        """
        # Extract semantic tokens
        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", generated_text)
        semantic_ids = [int(t) for t in semantic_matches]

        # Extract global tokens
        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
        global_ids = [int(t) for t in global_matches]

        # Only log if there's an issue
        if not semantic_ids and not global_ids:
            logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")

        return semantic_ids, global_ids


sed -n '1,330p' veena3modal/core/pipeline.py


Output:
Chunk ID: 4ca67e
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2946
Output:
"""
Spark TTS Generation Pipeline

End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
"""

import asyncio
import logging
import os
import re
from statistics import mean, median
from typing import Optional, List, Dict, Any, Tuple

from vllm import SamplingParams
import torch

logger = logging.getLogger(__name__)

from veena3modal.core.constants import (
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_SEED,
    AUDIO_SAMPLE_RATE,
)

def _detailed_timing_enabled() -> bool:
    """
    Return True when benchmark logs are enabled.

    Controlled by environment variable:
      VEENA3_TIMING_LOGS=true|1
    """
    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}

def _batch_wait_timeout_seconds() -> float:
    """Optional hard timeout (seconds) for waiting on each vLLM iterator batch."""
    raw = os.getenv("VEENA3_PIPELINE_BATCH_TIMEOUT_SECONDS", "").strip()
    if not raw:
        return 0.0
    try:
        return max(0.0, float(raw))
    except ValueError:
        return 0.0

class SparkTTSPipeline:
    """
    End-to-end TTS pipeline for Spark TTS with BiCodec.

    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        bicodec_decoder,
    ):
        """
        Initialize pipeline.

        Args:
            model: SparkTTS Model instance
            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
            bicodec_decoder: BiCodecDecoder instance
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.bicodec_decoder = bicodec_decoder

        print(f"🚀 SparkTTSPipeline initialized")

    async def generate_speech(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
        """
        Generate speech audio (non-streaming) using Spark TTS.

        NOTE: This method signature changed from description-based to speaker-based
        to align with Spark TTS architecture.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize (with optional [emotion] tags)
            temperature: Sampling temperature (default: 0.8 for Spark TTS)
            top_k: Top-k sampling (default: 50)
            top_p: Nucleus sampling (default: 1.0)
            max_tokens: Max BiCodec tokens to generate (default: 2048)
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Build prompt using Spark TTS format
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling for Spark TTS
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
        )

        # Use vLLM engine to generate
        if request_id is None:
            import uuid
            request_id = f"req-{uuid.uuid4().hex[:12]}"

        emit_timing = return_timing or _detailed_timing_enabled()
        batch_timeout = _batch_wait_timeout_seconds()
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Collect all outputs and track generation timing
        final_output = None
        token_batches: List[int] = []
        batch_wall_ms: List[float] = []
        batch_gpu_ms: List[float] = []
        total_token_ids = 0
        first_token_ms: Optional[float] = None
        t_start = asyncio.get_event_loop().time()
        if emit_timing:
            logger.info(
                "tts_llm_generate_start",
                extra={
                    "speaker": speaker,
                    "request_id": request_id,
                    "max_tokens": max_tokens,
                    "batch_timeout_seconds": batch_timeout,
                },
            )

        iterator = results_generator.__aiter__()
        while True:
            wall_start = asyncio.get_event_loop().time()
            start_event = None
            end_event = None
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                start_event = torch.cuda.Event(enable_timing=True)
                end_event = torch.cuda.Event(enable_timing=True)
                start_event.record()

            try:
                if batch_timeout:
                    request_output = await asyncio.wait_for(
                        iterator.__anext__(),
                        timeout=batch_timeout,
                    )
                else:
                    request_output = await iterator.__anext__()
            except asyncio.TimeoutError as exc:
                if emit_timing:
                    logger.error(
                        "tts_llm_batch_timeout",
                        extra={
                            "speaker": speaker,
                            "request_id": request_id,
                            "batch_index": len(token_batches) + 1,
                            "batch_timeout_seconds": batch_timeout,
                        },
                    )
                raise TimeoutError(
                    f"LLM batch wait timed out after {batch_timeout:.2f}s"
                ) from exc
            except StopAsyncIteration:
                break

            wall_end = asyncio.get_event_loop().time()
            batch_ms = (wall_end - wall_start) * 1000
            batch_wall_ms.append(batch_ms)

            batch_gpu_ms_value: Optional[float] = None
            if start_event is not None and end_event is not None:
                end_event.record()
                torch.cuda.synchronize()
                batch_gpu_ms_value = start_event.elapsed_time(end_event)
            if batch_gpu_ms_value is not None:
                batch_gpu_ms.append(batch_gpu_ms_value)

            total_generated = len(request_output.outputs[0].token_ids)
            new_tokens = max(0, total_generated - total_token_ids)
            total_token_ids = total_generated
            token_batches.append(new_tokens)

            if first_token_ms is None:
                first_token_ms = (wall_end - t_start) * 1000

            if emit_timing:
                logger.info(
                    "tts_llm_batch",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "batch_index": len(token_batches),
                        "batch_tokens": new_tokens,
                        "llm_batch_wall_ms": round(batch_ms, 3),
                        "llm_batch_gpu_ms": round(batch_gpu_ms_value, 3)
                        if batch_gpu_ms_value is not None
                        else None,
                    },
                )
            final_output = request_output

        if final_output is None:
            if emit_timing:
                logger.warning(
                    "tts_llm_no_output",
                    extra={"speaker": speaker, "request_id": request_id},
                )
            print(f"❌ Generation failed: no output")
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": 0,
                    "token_total": 0,
                    "batch_count": 0,
                }
            return None

        # Extract generated text (contains BiCodec tokens)
        parse_start = asyncio.get_event_loop().time()
        generated_text = final_output.outputs[0].text
        generated_ids = final_output.outputs[0].token_ids

        # Extract BiCodec tokens using regex
        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)

        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000

        if not semantic_ids or not global_ids:
            if emit_timing:
                logger.error(
                    "tts_bicodec_parse_failed",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "semantic_count": len(semantic_ids),
                        "global_count": len(global_ids),
                        "token_total": len(generated_ids),
                    },
                )
            logger.error(
                "❌ No BiCodec tokens found in output",
                extra={
                    "speaker": speaker,
                    "text_length": len(text),
                    "text_preview": text[:100],
                    "generated_tokens": len(generated_ids),
                    "semantic_tokens": len(semantic_ids),
                    "global_tokens": len(global_ids),
                    "generated_text_preview": generated_text[:500],
                }
            )
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Validate tokens
        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
            logger.error("❌ BiCodec token validation failed", extra={
                "speaker": speaker,
                "text_length": len(text),
                "semantic_tokens": len(semantic_ids),
                "global_tokens": len(global_ids),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Decode to audio
        decode_start = asyncio.get_event_loop().time()
        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000

        if audio_bytes is None:
            if emit_timing:
                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
            logger.error("❌ BiCodec decode failed", extra={
                "speaker": speaker,
                "text_length": len(text),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        total_ms = (asyncio.get_event_loop().time() - t_start) * 1000

        timing_metrics: Dict[str, Any] = {


ps -ef | rg 'local_server|uvicorn' | rg -v rg


Output:
Chunk ID: f957aa
Wall time: 0.0513 seconds
Process exited with code 1
Original token count: 0
Output:


Output:
Chunk ID: 10afeb
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 5
Output:
spark_tts_4speaker


I’m now moving to execution: I’ll start the local server with detailed timing enabled and then run the optimized stress script over increasing concurrency levels to generate per-request GPU timing artifacts and identify where generation stalls under load. I’ll only adjust the benchmark script if the default output misses the fields you asked for.


cd /home/ubuntu/veenaModal && VEENA3_TIMING_LOGS=true GPU_MEMORY_UTILIZATION=0.1 MODEL_PATH=$(pwd)/models/spark_tts_4speaker HF_TOKEN= python -m veena3modal.local_server --model-path $(pwd)/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000


Output:
Chunk ID: 958e1c
Wall time: 0.0503 seconds
Process exited with code 127
Original token count: 12
Output:
/bin/bash: line 1: python: command not found


cd /home/ubuntu/veenaModal && VEENA3_TIMING_LOGS=true GPU_MEMORY_UTILIZATION=0.1 MODEL_PATH=$(pwd)/models/spark_tts_4speaker HF_TOKEN= python3 -m veena3modal.local_server --model-path $(pwd)/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000


Output:
Chunk ID: 326476
Wall time: 1.0011 seconds
Process running with session ID 43127
Original token count: 458
Output:
2026-02-17 00:57:55 | INFO     | veena3.local | Auth bypass enabled (local mode). Use --auth to require API keys.
2026-02-17 00:57:55 | INFO     | veena3.local | Resolved model paths:
2026-02-17 00:57:55 | INFO     | veena3.local |   LLM (vLLM):    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:57:55 | INFO     | veena3.local |   BiCodec:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:57:55 | INFO     | veena3.local | ============================================================
2026-02-17 00:57:55 | INFO     | veena3.local | Initializing Veena3 TTS Runtime (local)
2026-02-17 00:57:55 | INFO     | veena3.local |   Model:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:57:55 | INFO     | veena3.local |   LLM path:    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:57:55 | INFO     | veena3.local |   BiCodec:     /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:57:55 | INFO     | veena3.local |   Device:      cuda
2026-02-17 00:57:55 | INFO     | veena3.local |   GPU Memory:  10% (total)
2026-02-17 00:57:55 | INFO     | veena3.local |   Engines:     1
2026-02-17 00:57:55 | INFO     | veena3.local |   SR:          disabled
2026-02-17 00:57:55 | INFO     | veena3.local | ============================================================
2026-02-17 00:57:55 | INFO     | veena3modal.services.tts_runtime | Initializing TTS runtime...
2026-02-17 00:57:55 | INFO     | veena3modal.services.tts_runtime |   Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:57:55 | INFO     | veena3modal.services.tts_runtime |   BiCodec path: /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:57:55 | INFO     | veena3modal.services.tts_runtime |   SR enabled: False, path: None


The local server started and is in the model-init phase; I’m polling until it’s ready, then I’ll run the benchmark sweep and capture the timing artifacts.


Output:
Chunk ID: dcc200
Wall time: 5.0010 seconds
Process running with session ID 43127
Original token count: 1295
Output:
2026-02-17 00:58:00 | INFO     | veena3modal.services.tts_runtime | Loading SparkTTS model with vLLM...
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.1
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
INFO 02-17 00:58:01 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 00:58:01 [model.py:1561] Using max model len 4096
INFO 02-17 00:58:01 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 00:58:01 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3722869) INFO 02-17 00:58:02 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}


Output:
Chunk ID: 3f5f59
Wall time: 2.6008 seconds
Process exited with code 1
Original token count: 4223
Output:
Total output lines: 164

(EngineCore_DP0 pid=3722869) INFO 02-17 00:58:04 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:36859 backend=nccl
(EngineCore_DP0 pid=3722869) INFO 02-17 00:58:04 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946] EngineCore failed to start.

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946] Traceback (most recent call last):

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 937, in run_engine_core

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 691, in __init__

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     super().__init__(

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 105, in __init__

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     self.model_executor = executor_class(vllm_config)

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/executor/abstract.py", line 101, in __init__

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     self._init_executor()

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/executor/uniproc_executor.py", line 47, in _init_executor

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     self.driver_worker.init_device()

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     self.worker.init_device()  # type: ignore

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/gpu_worker.py", line 258, in init_device

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     from vllm.v1.worker.gpu_model_runner import (

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/gpu_model_runner.py", line 153, in <module>

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     from vllm.v1.spec_decode.ngram_proposer import NgramProposer

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/spec_decode/ngram_proposer.py", line 170, in <module>

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     def batch_propose_numba(

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/decorators.py", line 221, in wrapper

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     disp = dispatcher(py_func=func, locals=locals,

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/dispatcher.py", line 772, in __init__

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     self.targetctx = self.targetdescr.target_context

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/registry.py", line 30, in target_context

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     return self._toplevel_target_context

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/utils.py", line 741, in __get__

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     return super().__get__(*args, **kwargs)

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/usr/lib/python3.10/functools.py", line 981, in __get__

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     val = self.func(instance)

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/registry.py", line 18, in _toplevel_target_context

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     return cpu.CPUContext(self.typing_context, self._target_name)

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/cpu.py", line 42, in __init__

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     super().__init__(typingctx, target)

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/base.py", line 254, in __init__

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     self.init()

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/compiler_lock.py", line 35, in _acquire_compile_lock

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     return func(*args, **kwargs)

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/cpu.py", line 51, in init

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     self._internal_codegen = codegen.JITCPUCodegen("numba.exec")

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/codegen.py", line 1172, in __init__

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     initialize_llvm()

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/codegen.py", line 1434, in initialize_llvm

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     ll.initialize()

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]   File "/home/ubuntu/.local/lib/python3.10/site-packages/llvmlite/binding/initfini.py", line 17, in initialize

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946]     raise RuntimeError(

(EngineCore_DP0 pid=3722869) ERROR 02-17 00:58:04 [core.py:946] RuntimeError: llvmlite.binding.initialize() is deprecated and will be removed. LLVM initialization is now handled automatically. Please remove calls to this function from your code and check for other behavioral changes that may have occurred due to LLVM updates.
(EngineCore_DP0 pid=3722869) Process EngineCore_DP0:
(EngineCore_DP0 pid=3722869) Traceback (most recent call last):
(EngineCore_DP0 pid=3722869)   File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
(EngineCore_DP0 pid=3722869)     self.run()
(EngineCore_DP0 pid=3722869)   File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
(EngineCore_DP0 pid=3722869)     self._target(*self._args, **self._kwargs)
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 950, in run_engine_core
(EngineCore_DP0 pid=3722869)     raise e
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core.py", line 937, in run_engine_core
(EngineCore_DP0 pid=3722869)     engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/v…223 tokens truncated…0 pid=3722869)     self.worker.init_device()  # type: ignore
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/gpu_worker.py", line 258, in init_device
(EngineCore_DP0 pid=3722869)     from vllm.v1.worker.gpu_model_runner import (
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/gpu_model_runner.py", line 153, in <module>
(EngineCore_DP0 pid=3722869)     from vllm.v1.spec_decode.ngram_proposer import NgramProposer
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/spec_decode/ngram_proposer.py", line 170, in <module>
(EngineCore_DP0 pid=3722869)     def batch_propose_numba(
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/decorators.py", line 221, in wrapper
(EngineCore_DP0 pid=3722869)     disp = dispatcher(py_func=func, locals=locals,
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/dispatcher.py", line 772, in __init__
(EngineCore_DP0 pid=3722869)     self.targetctx = self.targetdescr.target_context
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/registry.py", line 30, in target_context
(EngineCore_DP0 pid=3722869)     return self._toplevel_target_context
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/utils.py", line 741, in __get__
(EngineCore_DP0 pid=3722869)     return super().__get__(*args, **kwargs)
(EngineCore_DP0 pid=3722869)   File "/usr/lib/python3.10/functools.py", line 981, in __get__
(EngineCore_DP0 pid=3722869)     val = self.func(instance)
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/registry.py", line 18, in _toplevel_target_context
(EngineCore_DP0 pid=3722869)     return cpu.CPUContext(self.typing_context, self._target_name)
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/cpu.py", line 42, in __init__
(EngineCore_DP0 pid=3722869)     super().__init__(typingctx, target)
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/base.py", line 254, in __init__
(EngineCore_DP0 pid=3722869)     self.init()
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/compiler_lock.py", line 35, in _acquire_compile_lock
(EngineCore_DP0 pid=3722869)     return func(*args, **kwargs)
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/cpu.py", line 51, in init
(EngineCore_DP0 pid=3722869)     self._internal_codegen = codegen.JITCPUCodegen("numba.exec")
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/codegen.py", line 1172, in __init__
(EngineCore_DP0 pid=3722869)     initialize_llvm()
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/numba/core/codegen.py", line 1434, in initialize_llvm
(EngineCore_DP0 pid=3722869)     ll.initialize()
(EngineCore_DP0 pid=3722869)   File "/home/ubuntu/.local/lib/python3.10/site-packages/llvmlite/binding/initfini.py", line 17, in initialize
(EngineCore_DP0 pid=3722869)     raise RuntimeError(
(EngineCore_DP0 pid=3722869) RuntimeError: llvmlite.binding.initialize() is deprecated and will be removed. LLVM initialization is now handled automatically. Please remove calls to this function from your code and check for other behavioral changes that may have occurred due to LLVM updates.
2026-02-17 00:58:05 | ERROR    | veena3modal.services.tts_runtime | ❌ Failed to initialize TTS runtime: Engine core initialization failed. See root cause above. Failed core proc(s): {}
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 180, in initialize_runtime
    model = SparkTTSModel(
  File "/home/ubuntu/veenaModal/veena3modal/core/model_loader.py", line 128, in __init__
    self.engine = AsyncLLMEngine.from_engine_args(engine_args)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 257, in from_engine_args
    return cls(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 155, in __init__
    self.engine_core = EngineCoreClient.make_async_mp_client(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 122, in make_async_mp_client
    return AsyncMPClient(*client_args)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 819, in __init__
    super().__init__(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 479, in __init__
    with launch_core_engines(vllm_config, executor_class, log_stats) as (
  File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__
    next(self.gen)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/utils.py", line 933, in launch_core_engines
    wait_for_engine_startup(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/utils.py", line 992, in wait_for_engine_startup
    raise RuntimeError(
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 180, in initialize_runtime
    model = SparkTTSModel(
  File "/home/ubuntu/veenaModal/veena3modal/core/model_loader.py", line 128, in __init__
    self.engine = AsyncLLMEngine.from_engine_args(engine_args)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 257, in from_engine_args
    return cls(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 155, in __init__
    self.engine_core = EngineCoreClient.make_async_mp_client(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 122, in make_async_mp_client
    return AsyncMPClient(*client_args)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 819, in __init__
    super().__init__(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/core_client.py", line 479, in __init__
    with launch_core_engines(vllm_config, executor_class, log_stats) as (
  File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__
    next(self.gen)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/utils.py", line 933, in launch_core_engines
    wait_for_engine_startup(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/utils.py", line 992, in wait_for_engine_startup
    raise RuntimeError(
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/home/ubuntu/veenaModal/veena3modal/local_server.py", line 465, in <module>
    main()
  File "/home/ubuntu/veenaModal/veena3modal/local_server.py", line 426, in main
    initialize_local_runtime(
  File "/home/ubuntu/veenaModal/veena3modal/local_server.py", line 224, in initialize_local_runtime
    runtime = initialize_runtime(
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 285, in initialize_runtime
    raise RuntimeError(f"TTS runtime initialization failed: {e}") from e
RuntimeError: TTS runtime initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {}


Current init blocked by vLLM + numba startup (`llvmlite.binding.initialize` deprecation/runtime error). I’ll restart with `NUMBA_DISABLE_JIT=1` so we can complete warm-up and run stress runs against the optimized local path.


cd /home/ubuntu/veenaModal && VEENA3_TIMING_LOGS=true GPU_MEMORY_UTILIZATION=0.1 NUMBA_DISABLE_JIT=1 MODEL_PATH=$(pwd)/models/spark_tts_4speaker python3 -m veena3modal.local_server --model-path $(pwd)/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000


Output:
Chunk ID: 286002
Wall time: 1.0018 seconds
Process running with session ID 79955
Original token count: 458
Output:
2026-02-17 00:58:08 | INFO     | veena3.local | Auth bypass enabled (local mode). Use --auth to require API keys.
2026-02-17 00:58:08 | INFO     | veena3.local | Resolved model paths:
2026-02-17 00:58:08 | INFO     | veena3.local |   LLM (vLLM):    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:58:08 | INFO     | veena3.local |   BiCodec:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:58:08 | INFO     | veena3.local | ============================================================
2026-02-17 00:58:08 | INFO     | veena3.local | Initializing Veena3 TTS Runtime (local)
2026-02-17 00:58:08 | INFO     | veena3.local |   Model:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:58:08 | INFO     | veena3.local |   LLM path:    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:58:08 | INFO     | veena3.local |   BiCodec:     /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:58:08 | INFO     | veena3.local |   Device:      cuda
2026-02-17 00:58:08 | INFO     | veena3.local |   GPU Memory:  10% (total)
2026-02-17 00:58:08 | INFO     | veena3.local |   Engines:     1
2026-02-17 00:58:08 | INFO     | veena3.local |   SR:          disabled
2026-02-17 00:58:08 | INFO     | veena3.local | ============================================================
2026-02-17 00:58:08 | INFO     | veena3modal.services.tts_runtime | Initializing TTS runtime...
2026-02-17 00:58:08 | INFO     | veena3modal.services.tts_runtime |   Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:58:08 | INFO     | veena3modal.services.tts_runtime |   BiCodec path: /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:58:08 | INFO     | veena3modal.services.tts_runtime |   SR enabled: False, path: None


Output:
Chunk ID: 7a9a0d
Wall time: 5.0005 seconds
Process running with session ID 79955
Original token count: 1295
Output:
2026-02-17 00:58:13 | INFO     | veena3modal.services.tts_runtime | Loading SparkTTS model with vLLM...
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.1
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
INFO 02-17 00:58:14 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 00:58:14 [model.py:1561] Using max model len 4096
INFO 02-17 00:58:14 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 00:58:14 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:15 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}


Output:
Chunk ID: d3b0ac
Wall time: 5.0013 seconds
Process running with session ID 79955
Original token count: 440
Output:
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:18 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:40649 backend=nccl
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:18 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:18 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:19 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=3723091) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=3723091) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=3723091)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=3723091)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.36s/it]
(EngineCore_DP0 pid=3723091)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.36s/it]
(EngineCore_DP0 pid=3723091)
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:20 [default_loader.py:291] Loading weights took 1.44 seconds
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:21 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.967358 seconds


Output:
Chunk ID: ebd057
Wall time: 5.0008 seconds
Process running with session ID 79955
Original token count: 75
Output:
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:26 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:26 [backends.py:872] Dynamo bytecode transform time: 4.20 s


Output:
Chunk ID: 2c68a5
Wall time: 5.0008 seconds
Process running with session ID 79955
Original token count: 694
Output:
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:29 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.664 s
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:29 [monitor.py:34] torch.compile takes 4.87 s in total
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:29 [gpu_worker.py:356] Available KV cache memory: 6.17 GiB
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:29 [kv_cache_utils.py:1307] GPU KV cache size: 538,928 tokens
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:29 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 131.57x
(EngineCore_DP0 pid=3723091)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%| | 0/35 [00:00<?,
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%| | 4/35 [00:00<00:
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|▏| 8/35 [00:00<00:
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|▎| 12/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|▍| 16/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|▌| 20/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|▋| 24/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|▊| 28/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|▉| 32/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|█| 35/35 [00:01<00
(EngineCore_DP0 pid=3723091)
Capturing CUDA graphs (decode, FULL):   0%|              | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|▉     | 3/19 [00:00<00:00, 29.02it/s]
Capturing CUDA graphs (decode, FULL):  37%|██▏   | 7/19 [00:00<00:00, 33.50it/s]
Capturing CUDA graphs (decode, FULL):  58%|██▉  | 11/19 [00:00<00:00, 32.93it/s]
Capturing CUDA graphs (decode, FULL):  79%|███▉ | 15/19 [00:00<00:00, 34.24it/s]
Capturing CUDA graphs (decode, FULL): 100%|█████| 19/19 [00:00<00:00, 35.33it/s]
Capturing CUDA graphs (decode, FULL): 100%|█████| 19/19 [00:00<00:00, 34.28it/s]
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:32 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=3723091) INFO 02-17 00:58:32 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.86 seconds
✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

2026-02-17 00:58:33 | INFO     | veena3modal.services.tts_runtime | Initializing prompt builder...
2026-02-17 00:58:33 | INFO     | veena3modal.services.tts_runtime | Initializing BiCodec decoder...
🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cuda


Output:
Chunk ID: f357e1
Wall time: 5.0009 seconds
Process running with session ID 79955
Original token count: 1527
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
2026-02-17 00:58:39 | ERROR    | veena3modal.services.tts_runtime | ❌ Failed to initialize TTS runtime: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 195, in initialize_runtime
    bicodec_decoder = BiCodecDecoder(
  File "/home/ubuntu/veenaModal/veena3modal/core/bicodec_decoder.py", line 68, in __init__
    self.audio_tokenizer = BiCodecTokenizer(model_path, device=str(self.device))
  File "/home/ubuntu/veenaModal/external/sparktts/sparktts/models/audio_tokenizer.py", line 42, in __init__
    self._initialize_model()
  File "/home/ubuntu/veenaModal/external/sparktts/sparktts/models/audio_tokenizer.py", line 46, in _initialize_model
    self.model = BiCodec.load_from_checkpoint(f"{self.model_dir}/BiCodec").to(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1371, in to
    return self._apply(convert)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 930, in _apply
    module._apply(fn)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 930, in _apply
    module._apply(fn)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 930, in _apply
    module._apply(fn)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 957, in _apply
    param_applied = fn(param)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1357, in convert
    return t.to(
torch.AcceleratorError: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 195, in initialize_runtime
    bicodec_decoder = BiCodecDecoder(
  File "/home/ubuntu/veenaModal/veena3modal/core/bicodec_decoder.py", line 68, in __init__
    self.audio_tokenizer = BiCodecTokenizer(model_path, device=str(self.device))
  File "/home/ubuntu/veenaModal/external/sparktts/sparktts/models/audio_tokenizer.py", line 42, in __init__
    self._initialize_model()
  File "/home/ubuntu/veenaModal/external/sparktts/sparktts/models/audio_tokenizer.py", line 46, in _initialize_model
    self.model = BiCodec.load_from_checkpoint(f"{self.model_dir}/BiCodec").to(
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1371, in to
    return self._apply(convert)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 930, in _apply
    module._apply(fn)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 930, in _apply
    module._apply(fn)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 930, in _apply
    module._apply(fn)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 957, in _apply
    param_applied = fn(param)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1357, in convert
    return t.to(
torch.AcceleratorError: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/home/ubuntu/veenaModal/veena3modal/local_server.py", line 465, in <module>
    main()
  File "/home/ubuntu/veenaModal/veena3modal/local_server.py", line 426, in main
    initialize_local_runtime(
  File "/home/ubuntu/veenaModal/veena3modal/local_server.py", line 224, in initialize_local_runtime
    runtime = initialize_runtime(
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 285, in initialize_runtime
    raise RuntimeError(f"TTS runtime initialization failed: {e}") from e
RuntimeError: TTS runtime initialization failed: CUDA error: out of memory
Search for `cudaErrorMemoryAllocation' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

ERROR 02-17 00:58:39 [core_client.py:605] Engine core proc EngineCore_DP0 died unexpectedly, shutting down client.


sed -n '1,220p' veena3modal/core/bicodec_decoder.py


Output:
Chunk ID: 328f23
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2043
Output:
"""
Spark TTS BiCodec Decoder

Decodes BiCodec audio tokens (semantic + global) to audio waveforms.
Replaces SNAC decoder for Spark TTS model.
"""

import torch
import numpy as np
import sys
from pathlib import Path
from typing import List, Optional, Tuple

# Ensure `sparktts` (vendored under external/) is importable in local/dev runs.
# In Modal, PYTHONPATH already includes `/root/external/sparktts`.
try:
    from sparktts.models.audio_tokenizer import BiCodecTokenizer
except ImportError:  # pragma: no cover - environment-dependent
    _this_file = Path(__file__).resolve()
    _sparktts_path = None
    for _p in [_this_file.parent] + list(_this_file.parents):
        _candidate = _p / "external" / "sparktts"
        if _candidate.is_dir():
            _sparktts_path = str(_candidate)
            break
    if _sparktts_path and _sparktts_path not in sys.path:
        sys.path.insert(0, _sparktts_path)
    from sparktts.models.audio_tokenizer import BiCodecTokenizer

from veena3modal.core.constants import (
    BICODEC_TOKENIZER_PATH,
    AUDIO_SAMPLE_RATE,
)

class BiCodecDecoder:
    """
    BiCodec Decoder for Spark TTS.

    Decodes semantic and global tokens to audio waveforms using BiCodec.
    This replaces the SNAC decoder used in the old Indic Orpheus model.

    Supports streaming with sliding window approach (like SNAC).
    """

    def __init__(
        self,
        device: str = "cuda",
        model_path: str = BICODEC_TOKENIZER_PATH,
        enable_batching: bool = False,  # For compatibility with streaming pipeline
    ):
        """
        Initialize BiCodec decoder.

        Args:
            device: Device for BiCodec model (cuda/cpu)
            model_path: Path to BiCodec model checkpoint
            enable_batching: Enable async batching for streaming (not used yet)
        """
        self.device = torch.device(device if torch.cuda.is_available() else "cpu")
        self.model_path = model_path
        self.enable_batching = enable_batching  # For streaming pipeline compatibility

        print(f"🎵 Loading BiCodec audio tokenizer from {model_path}...")
        print(f"   Device: {self.device}")

        # Initialize BiCodec tokenizer
        self.audio_tokenizer = BiCodecTokenizer(model_path, device=str(self.device))

        # OPTIMIZATION: Ensure model is on device once at init, not per-decode call
        self.audio_tokenizer.device = self.device
        self.audio_tokenizer.model.to(self.device)

        # NOTE: torch.compile disabled for BiCodec -- dynamic semantic sequence lengths
        # cause repeated recompilation (~120s each), worse than eager mode.
        # torch.compile works best with static shapes. If needed, pre-warm with
        # representative lengths or use mode="default" instead of "reduce-overhead".
        # The multi-engine approach (Tier 3) provides more impactful gains.

        print(f"✅ BiCodec decoder initialized (sample rate: {AUDIO_SAMPLE_RATE}Hz)")
        if enable_batching:
            print(f"   ℹ  Batching enabled (for streaming support)")

    def decode(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
    ) -> Optional[np.ndarray]:
        """
        Decode BiCodec tokens to audio waveform.

        CRITICAL: BiCodec expects EXACTLY 32 global tokens!
        The decoder pools them via speaker_encoder → d_vector, then broadcasts
        via d_vector.unsqueeze(-1) across the time dimension.

        Args:
            semantic_ids: List of semantic token IDs (variable length)
            global_ids: List of global token IDs (MUST be exactly 32!)

        Returns:
            Audio waveform as numpy array (float32, 16kHz mono)
            Shape: (samples,)
            Returns None if decode fails
        """
        if not semantic_ids or not global_ids:
            print(f"⚠  Empty token lists: semantic={len(semantic_ids)}, global={len(global_ids)}")
            return None

        # Validate global token count
        EXPECTED_GLOBAL_TOKENS = 32
        if len(global_ids) != EXPECTED_GLOBAL_TOKENS:
            print(f"❌ BiCodec requires EXACTLY {EXPECTED_GLOBAL_TOKENS} global tokens, got {len(global_ids)}")
            # Try to fix it
            if len(global_ids) > EXPECTED_GLOBAL_TOKENS:
                global_ids = global_ids[:EXPECTED_GLOBAL_TOKENS]
                print(f"   └─ Truncated to {EXPECTED_GLOBAL_TOKENS}")
            else:
                global_ids = global_ids + [0] * (EXPECTED_GLOBAL_TOKENS - len(global_ids))
                print(f"   └─ Padded to {EXPECTED_GLOBAL_TOKENS}")

        try:
            # Convert to tensors following official Spark-TTS implementation
            # global_token_ids shape: (1, 32) - ALWAYS 32!
            # pred_semantic_ids shape: (1, N) - variable length
            pred_semantic = torch.tensor(semantic_ids).long().unsqueeze(0).to(self.device)
            pred_global = torch.tensor(global_ids).long().unsqueeze(0).to(self.device)

            # Device migration done once in __init__, not per-call
            # Decode through BiCodec
            # BiCodecTokenizer.detokenize() expects:
            #   - global_tokens: (batch, global_dim) -> will unsqueeze(1) internally
            #   - semantic_tokens: (batch, latent_dim)
            with torch.inference_mode():
                wav_np = self.audio_tokenizer.detokenize(
                    pred_global,  # (1, seq) - correct shape!
                    pred_semantic  # (1, seq)
                )

            return wav_np

        except Exception as e:
            print(f"❌ BiCodec decode error: {e}")
            import traceback
            traceback.print_exc()
            return None

    def decode_to_bytes(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
    ) -> Optional[bytes]:
        """
        Decode BiCodec tokens to audio bytes (int16 PCM).

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs

        Returns:
            Audio as bytes (int16 PCM, 16kHz mono)
            Returns None if decode fails
        """
        audio = self.decode(semantic_ids, global_ids)

        if audio is None:
            return None

        # Clip and convert float32 to int16 PCM
        audio = np.clip(audio, -1.0, 1.0)
        audio_int16 = (audio * 32767).astype(np.int16)

        return audio_int16.tobytes()

    def validate_tokens(self, semantic_ids: List[int], global_ids: List[int]) -> bool:
        """
        Validate BiCodec tokens before decoding.

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs

        Returns:
            True if valid, False otherwise
        """
        # Check minimum length
        if not semantic_ids:
            print(f"❌ No semantic tokens")
            return False

        if not global_ids:
            print(f"❌ No global tokens")
            return False

        # Basic sanity checks
        if len(semantic_ids) < 1:
            print(f"❌ Too few semantic tokens: {len(semantic_ids)}")
            return False

        if len(global_ids) < 1:
            print(f"❌ Too few global tokens: {len(global_ids)}")
            return False

        return True

    def get_audio_duration(self, semantic_ids: List[int]) -> float:
        """
        Estimate audio duration from semantic tokens.

        Args:
            semantic_ids: List of semantic token IDs

        Returns:
            Estimated duration in seconds
        """
        # BiCodec generates audio based on semantic tokens
        # Approximate duration calculation (may need tuning)
        # This is an estimate and actual duration depends on model output
        estimated_samples = len(semantic_ids) * 320  # Rough estimate
        return estimated_samples / AUDIO_SAMPLE_RATE


sed -n '1,240p' veena3modal/core/model_loader.py


Output:
Chunk ID: b7f9ae
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1800
Output:
"""
Spark TTS Model Loader

Loads Spark TTS model with vLLM engine and validates speakers/emotions.
Migrated from Veena3/Orpheus to Spark TTS architecture.
"""

import os
import torch
from typing import Optional, Dict, Any
from transformers import AutoTokenizer
from vllm import AsyncLLMEngine, AsyncEngineArgs, SamplingParams

from veena3modal.core.constants import (
    INDIC_EMOTION_TAGS,
    INDIC_SPEAKERS,
    SPEAKER_MAP,
    DEFAULT_MODEL_PATH,
    DEFAULT_MAX_MODEL_LEN,
    VLLM_CONFIG,
)

class SparkTTSModel:
    """
    Spark TTS Model with vLLM inference engine and BiCodec audio tokenizer.

    Model: BayAreaBoys/spark_tts_4speaker
    Architecture: Qwen2ForCausalLM with BiCodec audio tokenization
    Speakers: 12 predefined (mapped to speaker_0-11 as per training)
    Emotions: 10 tags in [bracket] format

    CRITICAL REQUIREMENTS:
    - Load tokenizer with special Spark TTS tokens
    - Configure vLLM with optimizations (prefix caching, CUDA graphs)
    - Support speaker mapping (lipakshi → speaker_0, etc.)
    """

    def __init__(
        self,
        model_path: str = None,
        model_type: str = "indic_speakers",  # Default to indic_speakers for backward compatibility
        hf_token: str = None,
        **engine_kwargs
    ):
        """
        Initialize Spark TTS model with vLLM.

        Args:
            model_path: Path to model directory (local or HF). If None, uses env var or default.
            model_type: Model type (kept for compatibility, always uses indic_speakers)
            hf_token: HuggingFace token for private models
            **engine_kwargs: Additional vLLM engine arguments (override defaults from VLLM_CONFIG)
        """
        self.model_type = "indic_speakers"  # Spark TTS only supports speaker-based

        # Use provided path, or check environment variable, or use default
        if model_path is None:
            model_path = os.environ.get(
                'SPARK_TTS_MODEL_PATH',
                DEFAULT_MODEL_PATH
            )

        self.model_path = model_path
        self.hf_token = hf_token

        print(f"🚀 Initializing Spark TTS Model")
        print(f"📁 Model path: {model_path}")
        print("🗣  Speakers: 12 predefined "
              "(lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, "
              "Nilay, Aarvi, Asha, Bittu, Mira)")
        print(f"🎭 Emotions: 10 tags in [bracket] format")

        # Load tokenizer
        print(f"\n📝 Loading tokenizer from model...")
        tokenizer_kwargs = {
            'trust_remote_code': True,
        }

        # Add HF token for private repos
        if hf_token:
            tokenizer_kwargs['token'] = hf_token

        # Add emotion tags as special tokens
        special_tokens = {
            'additional_special_tokens': INDIC_EMOTION_TAGS
        }

        self.tokenizer = AutoTokenizer.from_pretrained(
            model_path,
            **tokenizer_kwargs
        )

        # Add special tokens
        self.tokenizer.add_special_tokens(special_tokens)

        print(f"✅ Tokenizer loaded: {len(self.tokenizer)} tokens")

        # Validate speakers and emotions
        self._validate_speakers()
        self._validate_emotion_tags()

        # Initialize vLLM engine with Spark TTS optimizations
        print(f"\n🔧 Initializing vLLM engine with optimizations...")

        # Start with default VLLM_CONFIG and override with kwargs
        engine_kwargs_dict = VLLM_CONFIG.copy()
        engine_kwargs_dict.update({
            'model': model_path,
            'tokenizer': model_path,
        })

        # Add HF token for private models
        if hf_token:
            os.environ['HUGGING_FACE_HUB_TOKEN'] = hf_token

        # Override with any user-provided kwargs
        engine_kwargs_dict.update(engine_kwargs)

        # Print configuration
        print(f"  - dtype: {engine_kwargs_dict['dtype']}")
        print(f"  - max_model_len: {engine_kwargs_dict['max_model_len']}")
        print(f"  - gpu_memory_utilization: {engine_kwargs_dict['gpu_memory_utilization']}")
        print(f"  - enable_prefix_caching: {engine_kwargs_dict.get('enable_prefix_caching', False)}")
        print(f"  - enforce_eager (disable CUDA graphs): {engine_kwargs_dict.get('enforce_eager', True)}")

        engine_args = AsyncEngineArgs(**engine_kwargs_dict)
        self.engine = AsyncLLMEngine.from_engine_args(engine_args)

        print(f"✅ vLLM engine initialized successfully!")
        print(f"\n🎉 Spark TTS Model ready for inference!\n")

    def _validate_emotion_tags(self):
        """
        Validate that emotion tags are properly configured for Spark TTS.

        For Spark TTS, emotion tags use [bracket] format and should be
        added as special tokens to the tokenizer.
        """
        print(f"\n🔍 Validating {len(INDIC_EMOTION_TAGS)} emotion tags for Spark TTS...")

        # Check if emotion tags can be tokenized
        for tag in INDIC_EMOTION_TAGS:
            token_ids = self.tokenizer.encode(tag, add_special_tokens=False)
            # Spark TTS emotion tags may be multiple tokens, that's okay
            if not token_ids:
                print(f"⚠  Warning: Emotion tag {tag} produced no tokens")

        print(f"✅ All {len(INDIC_EMOTION_TAGS)} emotion tags validated")

        # Print sample emotion tags
        print(f"\n📋 Sample emotion tags:")
        for i, tag in enumerate(INDIC_EMOTION_TAGS[:5]):
            token_ids = self.tokenizer.encode(tag, add_special_tokens=False)
            print(f"  {tag}: {len(token_ids)} tokens")
        if len(INDIC_EMOTION_TAGS) > 5:
            print(f"  ...")

    def _validate_speakers(self):
        """
        Validate that all speakers are properly mapped for Spark TTS.

        Spark TTS uses speaker tokens like <|speaker_0|> to <|speaker_7|>.
        User-facing names (lipakshi, vardan, etc.) are mapped to these IDs.
        """
        print(f"\n🔍 Validating {len(INDIC_SPEAKERS)} speakers for Spark TTS...")

        # Validate speaker mapping
        for speaker_name in INDIC_SPEAKERS:
            if speaker_name not in SPEAKER_MAP:
                print(f"⚠  Warning: Speaker {speaker_name} not in SPEAKER_MAP")
                continue

            speaker_id = SPEAKER_MAP[speaker_name]
            speaker_token = f'<|speaker_{speaker_id}|>'
            token_ids = self.tokenizer.encode(speaker_token, add_special_tokens=False)
            print(f"  {speaker_name} → {speaker_token}: {len(token_ids)} tokens")

        print(f"✅ All {len(INDIC_SPEAKERS)} speakers validated with mapping")

    def get_model_type(self) -> str:
        """Get the model type (always 'indic_speakers' for Spark TTS)."""
        return self.model_type

    def get_tokenizer(self):
        """Get the tokenizer."""
        return self.tokenizer

    def get_engine(self):
        """Get the vLLM engine."""
        return self.engine


rg -n "BICODEC|bicodec_device|bicodec.*device|NUMBA_DISABLE_JIT" veena3modal -g '*.py'


Output:
Chunk ID: 0f1adf
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 160
Output:
veena3modal/services/tts_runtime.py:122:        bicodec_path: Path to BiCodec model (env: BICODEC_MODEL_PATH, defaults to model_path)
veena3modal/services/tts_runtime.py:146:        'BICODEC_MODEL_PATH',
veena3modal/core/bicodec_decoder.py:31:    BICODEC_TOKENIZER_PATH,
veena3modal/core/bicodec_decoder.py:49:        model_path: str = BICODEC_TOKENIZER_PATH,
veena3modal/core/constants.py:119:BICODEC_TOKENIZER_PATH = os.environ.get('MODEL_PATH', os.environ.get('SPARK_TTS_MODEL_PATH', '/models/spark_tts_4speaker'))
veena3modal/core/constants.py:163:BICODEC_BATCH_SIZE = 64
veena3modal/core/constants.py:164:BICODEC_BATCH_TIMEOUT_MS = 15


sed -n '1,260p' scripts/profile_pipeline.py