Output:
Chunk ID: df1a36
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 2233
Output:
"""
Spark TTS Generation Pipeline

End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
"""

import asyncio
import logging
import os
import re
from statistics import mean, median
from typing import Optional, List, Dict, Any, Tuple

from vllm import SamplingParams
import torch

logger = logging.getLogger(__name__)

from veena3modal.core.constants import (
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_SEED,
    AUDIO_SAMPLE_RATE,
)

def _detailed_timing_enabled() -> bool:
    """
    Return True when benchmark logs are enabled.

    Controlled by environment variable:
      VEENA3_TIMING_LOGS=true|1
    """
    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}

def _batch_wait_timeout_seconds() -> float:
    """Optional hard timeout (seconds) for waiting on each vLLM iterator batch."""
    raw = os.getenv("VEENA3_PIPELINE_BATCH_TIMEOUT_SECONDS", "").strip()
    if not raw:
        return 0.0
    try:
        return max(0.0, float(raw))
    except ValueError:
        return 0.0

class SparkTTSPipeline:
    """
    End-to-end TTS pipeline for Spark TTS with BiCodec.

    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        bicodec_decoder,
    ):
        """
        Initialize pipeline.

        Args:
            model: SparkTTS Model instance
            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
            bicodec_decoder: BiCodecDecoder instance
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.bicodec_decoder = bicodec_decoder

        print(f"🚀 SparkTTSPipeline initialized")

    async def generate_speech(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
        """
        Generate speech audio (non-streaming) using Spark TTS.

        NOTE: This method signature changed from description-based to speaker-based
        to align with Spark TTS architecture.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize (with optional [emotion] tags)
            temperature: Sampling temperature (default: 0.8 for Spark TTS)
            top_k: Top-k sampling (default: 50)
            top_p: Nucleus sampling (default: 1.0)
            max_tokens: Max BiCodec tokens to generate (default: 2048)
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Build prompt using Spark TTS format
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling for Spark TTS
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
        )

        # Use vLLM engine to generate
        if request_id is None:
            import uuid
            request_id = f"req-{uuid.uuid4().hex[:12]}"

        emit_timing = return_timing or _detailed_timing_enabled()
        batch_timeout = _batch_wait_timeout_seconds()
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Collect all outputs and track generation timing
        final_output = None
        token_batches: List[int] = []
        batch_wall_ms: List[float] = []
        batch_gpu_ms: List[float] = []
        total_token_ids = 0
        first_token_ms: Optional[float] = None
        t_start = asyncio.get_event_loop().time()
        if emit_timing:
            logger.info(
                "tts_llm_generate_start",
                extra={
                    "speaker": speaker,
                    "request_id": request_id,
                    "max_tokens": max_tokens,
                    "batch_timeout_seconds": batch_timeout,
                },
            )

        iterator = results_generator.__aiter__()
        while True:
            wall_start = asyncio.get_event_loop().time()
            start_event = None
            end_event = None
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                start_event = torch.cuda.Event(enable_timing=True)
                end_event = torch.cuda.Event(enable_timing=True)
                start_event.record()

            try:
                if batch_timeout:
                    request_output = await asyncio.wait_for(
                        iterator.__anext__(),
                        timeout=batch_timeout,
                    )
                else:
                    request_output = await iterator.__anext__()
            except asyncio.TimeoutError as exc:
                if emit_timing:
                    logger.error(
                        "tts_llm_batch_timeout",
                        extra={
                            "speaker": speaker,
                            "request_id": request_id,
                            "batch_index": len(token_batches) + 1,
                            "batch_timeout_seconds": batch_timeout,
                        },
                    )
                raise TimeoutError(
                    f"LLM batch wait timed out after {batch_timeout:.2f}s"
                ) from exc
            except StopAsyncIteration:
                break

            wall_end = asyncio.get_event_loop().time()
            batch_ms = (wall_end - wall_start) * 1000
            batch_wall_ms.append(batch_ms)

            batch_gpu_ms_value: Optional[float] = None
            if start_event is not None and end_event is not None:
                end_event.record()
                torch.cuda.synchronize()
                batch_gpu_ms_value = start_event.elapsed_time(end_event)
            if batch_gpu_ms_value is not None:
                batch_gpu_ms.append(batch_gpu_ms_value)

            total_generated = len(request_output.outputs[0].token_ids)
            new_tokens = max(0, total_generated - total_token_ids)
            total_token_ids = total_generated
            token_batches.append(new_tokens)

            if first_token_ms is None:
                first_token_ms = (wall_end - t_start) * 1000

            if emit_timing:
                logger.info(
                    "tts_llm_batch",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "batch_index": len(token_batches),
                        "batch_tokens": new_tokens,
                        "llm_batch_wall_ms": round(batch_ms, 3),
                        "llm_batch_gpu_ms": round(batch_gpu_ms_value, 3)
                        if batch_gpu_ms_value is not None
                        else None,
                    },
                )
            final_output = request_output

        if final_output is None:
            if emit_timing:
                logger.warning(
                    "tts_llm_no_output",
                    extra={"speaker": speaker, "request_id": request_id},
                )
            print(f"❌ Generation failed: no output")
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": 0,
                    "token_total": 0,
                    "batch_count": 0,
                }
            return None

        # Extract generated text (contains BiCodec tokens)
        parse_start = asyncio.get_event_loop().time()
        generated_text = final_output.outputs[0].text
        generated_ids = final_output.outputs[0].token_ids

        # Extract BiCodec tokens using regex
        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)

        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000

        if not semantic_ids or not global_ids:
            if emit_timing:
                logger.error(
                    "tts_bicodec_parse_failed",


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '260,520p' veena3modal/core/pipeline.py


Output:
Chunk ID: 153909
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1868
Output:
                    "tts_bicodec_parse_failed",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "semantic_count": len(semantic_ids),
                        "global_count": len(global_ids),
                        "token_total": len(generated_ids),
                    },
                )
            logger.error(
                "❌ No BiCodec tokens found in output",
                extra={
                    "speaker": speaker,
                    "text_length": len(text),
                    "text_preview": text[:100],
                    "generated_tokens": len(generated_ids),
                    "semantic_tokens": len(semantic_ids),
                    "global_tokens": len(global_ids),
                    "generated_text_preview": generated_text[:500],
                }
            )
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Validate tokens
        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
            logger.error("❌ BiCodec token validation failed", extra={
                "speaker": speaker,
                "text_length": len(text),
                "semantic_tokens": len(semantic_ids),
                "global_tokens": len(global_ids),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Decode to audio
        decode_start = asyncio.get_event_loop().time()
        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000

        if audio_bytes is None:
            if emit_timing:
                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
            logger.error("❌ BiCodec decode failed", extra={
                "speaker": speaker,
                "text_length": len(text),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        total_ms = (asyncio.get_event_loop().time() - t_start) * 1000

        timing_metrics: Dict[str, Any] = {
            "request_id": request_id,
            "speaker": speaker,
            "text_length": len(text),
            "batch_count": len(token_batches),
            "token_total": len(generated_ids),
            "token_batches": token_batches,
            "ttfb_ms": int(first_token_ms or 0),
            "llm_wall_ms": int(sum(batch_wall_ms)),
            "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
            "llm_gpu_batch_ms_total": int(sum(batch_gpu_ms)),
            "llm_gpu_batch_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_p50": int(median(batch_gpu_ms)) if batch_gpu_ms else 0,
            "decode_cpu_ms": int(decode_ms),
            "parse_ms": int(parse_ms),
            "total_ms": int(total_ms),
        }
        if token_batches:
            timing_metrics["tokens_per_batch_min"] = min(token_batches)
            timing_metrics["tokens_per_batch_max"] = max(token_batches)
            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))

        if emit_timing:
            logger.info("tts_llm_complete", extra=timing_metrics)

        from veena3modal.audio.utils import add_wav_header
        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)

        if return_timing:
            return wav_bytes, timing_metrics
        return wav_bytes

    async def generate_speech_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ):
        """
        Generate speech audio for Spark TTS (speaker-based).

        This method is kept for backward compatibility with existing API.
        It delegates to generate_speech() with the same implementation.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "Hello <laugh> this is fun!" (will be normalized)
                - "नमस्ते! [excited] आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max BiCodec tokens to generate
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Delegate to generate_speech() with the same implementation for Spark TTS
        return await self.generate_speech(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
            return_timing=return_timing,
            request_id=request_id,
        )

    def _extract_bicodec_tokens(self, generated_text: str) -> Tuple[List[int], List[int]]:
        """
        Extract BiCodec semantic and global tokens from generated text using regex.

        Spark TTS generates tokens in the format:
        - <|bicodec_semantic_{id}|>
        - <|bicodec_global_{id}|>

        Args:
            generated_text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
            Returns ([], []) if no tokens found
        """
        # Extract semantic tokens
        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", generated_text)
        semantic_ids = [int(t) for t in semantic_matches]

        # Extract global tokens
        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
        global_ids = [int(t) for t in global_matches]

        # Only log if there's an issue
        if not semantic_ids and not global_ids:
            logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")

        return semantic_ids, global_ids


Summary: auto


curl -sS -m 120 -N -o /tmp/tts_stream.wav -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": false, "max_tokens": 128, "stream": true}'


Output:
Chunk ID: c5cd9c
Wall time: 30.0011 seconds
Process running with session ID 61212
Original token count: 0
Output:


Output:
Chunk ID: a31c74
Wall time: 10.0012 seconds
Process running with session ID 98130
Original token count: 149
Output:
INFO:     127.0.0.1:35420 - "GET /v1/tts/metrics HTTP/1.1" 200 OK
{"timestamp": "2026-02-17T01:14:06.470292+00:00", "level": "INFO", "logger": "veena3modal.tts", "message": "request_received", "request_id": "8138edea-96c2-4afb-b2c1-eeb12dafe4a0", "event": "request_received", "text_length": 5, "speaker": "lipakshi", "stream": true, "format": "wav"}
2026-02-17 01:14:06 | INFO     | veena3modal.tts | request_received
INFO:     127.0.0.1:45646 - "POST /v1/tts/generate HTTP/1.1" 200 OK
2026-02-17 01:14:06 | INFO     | veena3modal.processing.long_text_processor | LongTextProcessor initialized


curl -sS http://localhost:8000/v1/tts/metrics | rg 'requests_total|requests_completed_total|requests_failed_total'


Output:
Chunk ID: 304a6c
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 133
Output:
# HELP veena3_tts_requests_total Total TTS requests received
# TYPE veena3_tts_requests_total counter
veena3_tts_requests_total{format="AudioFormat_WAV",speaker="lipakshi",stream="false"} 1.0
veena3_tts_requests_total{format="AudioFormat_WAV",speaker="lipakshi",stream="true"} 1.0
# HELP veena3_tts_requests_completed_total Total TTS requests completed successfully
# TYPE veena3_tts_requests_completed_total counter
# HELP veena3_tts_requests_failed_total Total TTS requests failed
# TYPE veena3_tts_requests_failed_total counter


I found the root symptom: both non-stream and stream requests are accepted and never progress to completion (no `tts_llm_complete` / request_completed), so requests sit until client timeout. I’m testing whether this is tied to vLLM GPU memory throttling by restarting with a larger `--gpu-memory` and enforcing per-batch timeout hooks for deterministic failure signals.


pgrep -af 'python3 -m veena3modal.local_server|VLLM::EngineCore'


Output:
Chunk ID: c99aa3
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 77
Output:
3730907 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000
3731006 VLLM::EngineCore


Output:
Chunk ID: e3aa17
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 0
Output:


NUMBA_DISABLE_JIT=1 BICODEC_DEVICE=cpu VEENA3_PIPELINE_BATCH_TIMEOUT_SECONDS=20 VEENA3_REQUEST_TIMEOUT_SECONDS=120 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.5 --log-level INFO --port 8000


Output:
Chunk ID: 5d2530
Wall time: 30.0013 seconds
Process running with session ID 72757
Original token count: 3040
Output:
Total output lines: 95

2026-02-17 01:14:56 | INFO     | veena3.local | Auth bypass enabled (local mode). Use --auth to require API keys.
2026-02-17 01:14:57 | INFO     | veena3.local | Resolved model paths:
2026-02-17 01:14:57 | INFO     | veena3.local |   LLM (vLLM):    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:14:57 | INFO     | veena3.local |   BiCodec:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:14:57 | INFO     | veena3.local | ============================================================
2026-02-17 01:14:57 | INFO     | veena3.local | Initializing Veena3 TTS Runtime (local)
2026-02-17 01:14:57 | INFO     | veena3.local |   Model:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:14:57 | INFO     | veena3.local |   LLM path:    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:14:57 | INFO     | veena3.local |   BiCodec:     /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:14:57 | INFO     | veena3.local |   Device:      cuda
2026-02-17 01:14:57 | INFO     | veena3.local |   GPU Memory:  50% (total)
2026-02-17 01:14:57 | INFO     | veena3.local |   Engines:     1
2026-02-17 01:14:57 | INFO     | veena3.local |   SR:          disabled
2026-02-17 01:14:57 | INFO     | veena3.local | ============================================================
2026-02-17 01:14:57 | INFO     | veena3modal.services.tts_runtime | Initializing TTS runtime...
2026-02-17 01:14:57 | INFO     | veena3modal.services.tts_runtime |   Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:14:57 | INFO     | veena3modal.services.tts_runtime |   BiCodec path: /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:14:57 | INFO     | veena3modal.services.tts_runtime |   BiCodec device: cpu
2026-02-17 01:14:57 | INFO     | veena3modal.services.tts_runtime |   SR enabled: False, path: None
2026-02-17 01:15:02 | INFO     | veena3modal.services.tts_runtime | Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.5
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-17 01:15:03 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 01:15:03 [model.py:1561] Using max model len 4096
INFO 02-17 01:15:03 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 01:15:03 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:04 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'co…40 tokens truncated…_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:06 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:36775 backend=nccl
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:06 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:06 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:07 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=3732555) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=3732555) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=3732555)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=3732555)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.98it/s]
(EngineCore_DP0 pid=3732555)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.97it/s]
(EngineCore_DP0 pid=3732555)
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:08 [default_loader.py:291] Loading weights took 0.58 seconds
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:09 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.110571 seconds
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:13 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:13 [backends.py:872] Dynamo bytecode transform time: 4.16 s
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:16 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.634 s
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:16 [monitor.py:34] torch.compile takes 4.80 s in total
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:17 [gpu_worker.py:356] Available KV cache memory: 37.87 GiB
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:17 [kv_cache_utils.py:1307] GPU KV cache size: 3,309,040 tokens
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:17 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 807.87x
(EngineCore_DP0 pid=3732555)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 33.29it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 35.61it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 36.88it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 36.93it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 36.79it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 36.86it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 36.25it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 34.90it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:00<00:00, 35.07it/s]
(EngineCore_DP0 pid=3732555)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  21%|██        | 4/19 [00:00<00:00, 31.56it/s]
Capturing CUDA graphs (decode, FULL):  42%|████▏     | 8/19 [00:00<00:00, 34.45it/s]
Capturing CUDA graphs (decode, FULL):  63%|██████▎   | 12/19 [00:00<00:00, 34.42it/s]
Capturing CUDA graphs (decode, FULL):  84%|████████▍ | 16/19 [00:00<00:00, 35.15it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.08it/s]
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:19 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=3732555) INFO 02-17 01:15:19 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.71 seconds
2026-02-17 01:15:21 | INFO     | veena3modal.services.tts_runtime | Initializing prompt builder...
2026-02-17 01:15:21 | INFO     | veena3modal.services.tts_runtime | Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


Output:
Chunk ID: 9d8a71
Wall time: 30.0010 seconds
Process running with session ID 72757
Original token count: 1013
Output:
2026-02-17 01:15:29 | INFO     | veena3modal.services.tts_runtime | Initializing TTS pipeline...
2026-02-17 01:15:29 | INFO     | veena3modal.services.tts_runtime | Initializing streaming pipeline...
2026-02-17 01:15:29 | INFO     | veena3modal.services.tts_runtime | ✅ TTS runtime initialized in 32321ms
2026-02-17 01:15:29 | INFO     | veena3modal.services.tts_runtime |    Model version: LLM
2026-02-17 01:15:29 | INFO     | veena3modal.services.tts_runtime | Pre-computing global tokens for all speakers...
2026-02-17 01:15:29 | INFO     | veena3modal.services.tts_runtime |   Cached globals for lipakshi
2026-02-17 01:15:30 | INFO     | veena3modal.services.tts_runtime |   Cached globals for vardan
2026-02-17 01:15:30 | INFO     | veena3modal.services.tts_runtime |   Cached globals for reet
2026-02-17 01:15:31 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nandini
2026-02-17 01:15:31 | INFO     | veena3modal.services.tts_runtime |   Cached globals for krishna
2026-02-17 01:15:31 | INFO     | veena3modal.services.tts_runtime |   Cached globals for anika
2026-02-17 01:15:32 | INFO     | veena3modal.services.tts_runtime |   Cached globals for adarsh
2026-02-17 01:15:32 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nilay
2026-02-17 01:15:33 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Aarvi
2026-02-17 01:15:33 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Asha
2026-02-17 01:15:33 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Bittu
2026-02-17 01:15:34 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Mira
2026-02-17 01:15:34 | INFO     | veena3modal.services.tts_runtime | Speaker globals cached: 12/12 speakers in 4976ms
2026-02-17 01:15:34 | INFO     | veena3.local | Runtime initialized in 37.3s
2026-02-17 01:15:34 | INFO     | veena3.local |   Model version: LLM
2026-02-17 01:15:34 | INFO     | veena3.local |   Streaming: available
2026-02-17 01:15:34 | INFO     | veena3.local |   Super-resolution: disabled
2026-02-17 01:15:34 | INFO     | veena3.local |
2026-02-17 01:15:34 | INFO     | veena3.local | ============================================================
2026-02-17 01:15:34 | INFO     | veena3.local | Veena3 TTS Local Server starting
2026-02-17 01:15:34 | INFO     | veena3.local |   URL: http://0.0.0.0:8000
2026-02-17 01:15:34 | INFO     | veena3.local |   Health: http://0.0.0.0:8000/v1/tts/health
2026-02-17 01:15:34 | INFO     | veena3.local |   Generate: POST http://0.0.0.0:8000/v1/tts/generate
2026-02-17 01:15:34 | INFO     | veena3.local |   Metrics: http://0.0.0.0:8000/v1/tts/metrics
2026-02-17 01:15:34 | INFO     | veena3.local |   Docs: http://0.0.0.0:8000/docs
2026-02-17 01:15:34 | INFO     | veena3.local | ============================================================
✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cpu
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
🚀 SparkTTSPipeline initialized
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)
{"timestamp": "2026-02-17T01:15:34.360899+00:00", "level": "INFO", "logger": "veena3modal.api.websocket_handler", "message": "WebSocket route /v1/tts/ws registered"}
2026-02-17 01:15:34 | INFO     | veena3modal.api.websocket_handler | WebSocket route /v1/tts/ws registered
2026-02-17 01:15:34 | INFO     | veena3modal.api.fastapi_app | WebSocket support enabled at /v1/tts/ws
{"timestamp": "2026-02-17T01:15:34.361080+00:00", "level": "INFO", "logger": "veena3modal.api.fastapi_app", "message": "WebSocket support enabled at /v1/tts/ws"}
INFO:     Started server process [3732447]
INFO:     Waiting for application startup.
INFO:     Application startup complete.
INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)


curl -sS -m 180 -o /tmp/tts_one.wav -D - -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": false, "max_tokens": 128}'


Output:
Chunk ID: 0c60bd
Wall time: 20.1400 seconds
Process exited with code 0
Original token count: 50
Output:
HTTP/1.1 500 Internal Server Error
date: Tue, 17 Feb 2026 01:15:59 GMT
server: uvicorn
x-request-id: df7ac9e8-7c69-4229-953d-bdee9f214dc5
content-length: 137
content-type: application/json


Output:
Chunk ID: 5a62c0
Wall time: 5.0007 seconds
Process running with session ID 72757
Original token count: 1750
Output:
{"timestamp": "2026-02-17T01:15:59.700774+00:00", "level": "INFO", "logger": "veena3modal.tts", "message": "request_received", "request_id": "df7ac9e8-7c69-4229-953d-bdee9f214dc5", "event": "request_received", "text_length": 5, "speaker": "lipakshi", "stream": false, "format": "wav"}
2026-02-17 01:15:59 | INFO     | veena3modal.tts | request_received
2026-02-17 01:15:59 | INFO     | veena3modal.services.tts_runtime | tts_runtime_nonstream_start
2026-02-17 01:16:19 | ERROR    | veena3modal.services.tts_runtime | Speech generation failed: LLM batch wait timed out after 20.00s
2026-02-17 01:16:19 | ERROR    | veena3modal.services.tts_runtime | tts_runtime_nonstream_error
Traceback (most recent call last):
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 572, in generate
    out = q.get_nowait() or await q.get()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/output_processor.py", line 76, in get
    await self.ready.wait()
  File "/usr/lib/python3.10/asyncio/locks.py", line 214, in wait
    await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.10/asyncio/tasks.py", line 456, in wait_for
    return fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 172, in generate_speech
    request_output = await asyncio.wait_for(
  File "/usr/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
    raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 502, in generate_speech
    audio_bytes = await asyncio.wait_for(
  File "/usr/lib/python3.10/asyncio/tasks.py", line 445, in wait_for
    return fut.result()
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 401, in generate_speech_indic
    return await self.generate_speech(
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 189, in generate_speech
    raise TimeoutError(
TimeoutError: LLM batch wait timed out after 20.00s
{"timestamp": "2026-02-17T01:16:19.902066+00:00", "level": "ERROR", "logger": "veena3modal.api.fastapi_app", "message": "TTS generation error: LLM batch wait timed out after 20.00s", "request_id": "df7ac9e8-7c69-4229-953d-bdee9f214dc5", "exception": "Traceback (most recent call last):\n  File \"/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py\", line 572, in generate\n    out = q.get_nowait() or await q.get()\n  File \"/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/output_processor.py\", line 76, in get\n    await self.ready.wait()\n  File \"/usr/lib/python3.10/asyncio/locks.py\", line 214, in wait\n    await fut\nasyncio.exceptions.CancelledError\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n    return fut.result()\nasyncio.exceptions.CancelledError\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/home/ubuntu/veenaModal/veena3modal/core/pipeline.py\", line 172, in generate_speech\n    request_output = await asyncio.wait_for(\n  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n    raise exceptions.TimeoutError() from exc\nasyncio.exceptions.TimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py\", line 460, in tts_generate\n    audio_bytes, metrics = await tts_runtime.generate_speech(\n  File \"/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\", line 502, in generate_speech\n    audio_bytes = await asyncio.wait_for(\n  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 445, in wait_for\n    return fut.result()\n  File \"/home/ubuntu/veenaModal/veena3modal/core/pipeline.py\", line 401, in generate_speech_indic\n    return await self.generate_speech(\n  File \"/home/ubuntu/veenaModal/veena3modal/core/pipeline.py\", line 189, in generate_speech\n    raise TimeoutError(\nTimeoutError: LLM batch wait timed out after 20.00s\n"}
2026-02-17 01:16:19 | ERROR    | veena3modal.api.fastapi_app | TTS generation error: LLM batch wait timed out after 20.00s
Traceback (most recent call last):
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 572, in generate
    out = q.get_nowait() or await q.get()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/output_processor.py", line 76, in get
    await self.ready.wait()
  File "/usr/lib/python3.10/asyncio/locks.py", line 214, in wait
    await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.10/asyncio/tasks.py", line 456, in wait_for
    return fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 172, in generate_speech
    request_output = await asyncio.wait_for(
  File "/usr/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
    raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py", line 460, in tts_generate
    audio_bytes, metrics = await tts_runtime.generate_speech(
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 502, in generate_speech
    audio_bytes = await asyncio.wait_for(
  File "/usr/lib/python3.10/asyncio/tasks.py", line 445, in wait_for
    return fut.result()
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 401, in generate_speech_indic
    return await self.generate_speech(
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 189, in generate_speech
    raise TimeoutError(
TimeoutError: LLM batch wait timed out after 20.00s
{"timestamp": "2026-02-17T01:16:19.902570+00:00", "level": "ERROR", "logger": "veena3modal.tts", "message": "request_failed", "request_id": "df7ac9e8-7c69-4229-953d-bdee9f214dc5", "event": "request_failed", "status_code": 500, "error_code": "INTERNAL_ERROR", "error_message": "LLM batch wait timed out after 20.00s"}
2026-02-17 01:16:19 | ERROR    | veena3modal.tts | request_failed
INFO:     127.0.0.1:56610 - "POST /v1/tts/generate HTTP/1.1" 500 Internal Server Error


Output:
Chunk ID: 2a0ddf
Wall time: 0.0502 seconds
Process exited with code 0
Original token count: 77
Output:
3732447 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.5 --log-level INFO --port 8000
3732555 VLLM::EngineCore


Output:
Chunk ID: 30ddd5
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 0
Output:


NUMBA_DISABLE_JIT=1 BICODEC_DEVICE=cpu VEENA3_TIMING_LOGS=true python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.5 --log-level INFO --port 8000


Output:
Chunk ID: d3f006
Wall time: 30.0004 seconds
Process running with session ID 96712
Original token count: 3065
Output:
Total output lines: 95

2026-02-17 01:16:35 | INFO     | veena3.local | Auth bypass enabled (local mode). Use --auth to require API keys.
2026-02-17 01:16:35 | INFO     | veena3.local | Resolved model paths:
2026-02-17 01:16:35 | INFO     | veena3.local |   LLM (vLLM):    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:16:35 | INFO     | veena3.local |   BiCodec:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:16:35 | INFO     | veena3.local | ============================================================
2026-02-17 01:16:35 | INFO     | veena3.local | Initializing Veena3 TTS Runtime (local)
2026-02-17 01:16:35 | INFO     | veena3.local |   Model:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:16:35 | INFO     | veena3.local |   LLM path:    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:16:35 | INFO     | veena3.local |   BiCodec:     /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:16:35 | INFO     | veena3.local |   Device:      cuda
2026-02-17 01:16:35 | INFO     | veena3.local |   GPU Memory:  50% (total)
2026-02-17 01:16:35 | INFO     | veena3.local |   Engines:     1
2026-02-17 01:16:35 | INFO     | veena3.local |   SR:          disabled
2026-02-17 01:16:35 | INFO     | veena3.local | ============================================================
2026-02-17 01:16:35 | INFO     | veena3modal.services.tts_runtime | Initializing TTS runtime...
2026-02-17 01:16:35 | INFO     | veena3modal.services.tts_runtime |   Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:16:35 | INFO     | veena3modal.services.tts_runtime |   BiCodec path: /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:16:35 | INFO     | veena3modal.services.tts_runtime |   BiCodec device: cpu
2026-02-17 01:16:35 | INFO     | veena3modal.services.tts_runtime |   SR enabled: False, path: None
2026-02-17 01:16:40 | INFO     | veena3modal.services.tts_runtime | Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.5
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-17 01:16:41 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 01:16:41 [model.py:1561] Using max model len 4096
INFO 02-17 01:16:41 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 01:16:41 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:42 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'co…65 tokens truncated…hMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:44 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:35539 backend=nccl
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:44 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:45 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:46 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=3733081) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=3733081) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=3733081)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=3733081)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.67it/s]
(EngineCore_DP0 pid=3733081)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.67it/s]
(EngineCore_DP0 pid=3733081)
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:46 [default_loader.py:291] Loading weights took 0.66 seconds
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:47 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.194382 seconds
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:52 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:52 [backends.py:872] Dynamo bytecode transform time: 4.25 s
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:55 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.680 s
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:55 [monitor.py:34] torch.compile takes 4.93 s in total
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:56 [gpu_worker.py:356] Available KV cache memory: 37.87 GiB
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:56 [kv_cache_utils.py:1307] GPU KV cache size: 3,309,040 tokens
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:56 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 807.87x
(EngineCore_DP0 pid=3733081)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 32.84it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 34.84it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 35.09it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 36.04it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 36.18it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 36.08it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 35.67it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 34.31it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:01<00:00, 34.37it/s]
(EngineCore_DP0 pid=3733081)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 27.43it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 32.23it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 34.15it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 34.20it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 34.62it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 33.82it/s]
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:58 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=3733081) INFO 02-17 01:16:58 [core.py:272] init engine (profile, create kv cache, warmup model) took 11.02 seconds
2026-02-17 01:17:00 | INFO     | veena3modal.services.tts_runtime | Initializing prompt builder...
2026-02-17 01:17:00 | INFO     | veena3modal.services.tts_runtime | Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


Output:
Chunk ID: 5c0a12
Wall time: 30.0008 seconds
Process running with session ID 96712
Original token count: 1013
Output:
Total output lines: 50

2026-02-17 01:17:05 | INFO     | veena3modal.services.tts_runtime | Initializing TTS pipeline...
2026-02-17 01:17:05 | INFO     | veena3modal.services.tts_runtime | Initializing streaming pipeline...
2026-02-17 01:17:05 | INFO     | veena3modal.services.tts_runtime | ✅ TTS runtime initialized in 30443ms
2026-02-17 01:17:05 | INFO     | veena3modal.services.tts_runtime |    Model version: LLM
2026-02-17 01:17:05 | INFO     | veena3modal.services.tts_runtime | Pre-computing global tokens for all speakers...
2026-02-17 01:17:06 | INFO     | veena3modal.services.tts_runtime |   Cached globals for lipakshi
2026-02-17 01:17:06 | INFO     | veena3modal.services.tts_runtime |   Cached globals for vardan
2026-02-17 01:17:06 | INFO     | veena3modal.services.tts_runtime |   Cached globals for reet
2026-02-17 01:17:07 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nandini
2026-02-17 01:17:07 | INFO     | veena3modal.services.tts_runtime |   Cached globals for krishna
2026-02-17 01:17:08 | INFO     | veena3modal.services.tts_runtime |   Cached globals for anika
2026-02-17 01:17:08 | INFO     | veena3modal.services.tts_runtime |   Cached globals for adarsh
2026-02-17 01:17:08 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nilay
2026-02-17 01:17:09 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Aarvi
2026-02-17 01:17:09 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Asha
2026-02-17 01:17:10 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Bittu
2026-02-17 01:17:10 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Mira
2026-02-17 01:17:10 | INFO     | veena3modal.services.tts_runtime | Speaker globals cached: 12/12 speakers in 4928ms
2026-02-17 01:17:10 | INFO     | veena3.local | Runtime initialized in 35.4s
2026-02-17 01:17:10 | INFO     | veena3.local |   Model version: LLM
2026-02-17 01:17:10 | INFO     | veena3.local |   Streaming: available
2026-02…13 tokens truncated…solution: disabled
2026-02-17 01:17:10 | INFO     | veena3.local |
2026-02-17 01:17:10 | INFO     | veena3.local | ============================================================
2026-02-17 01:17:10 | INFO     | veena3.local | Veena3 TTS Local Server starting
2026-02-17 01:17:10 | INFO     | veena3.local |   URL: http://0.0.0.0:8000
2026-02-17 01:17:10 | INFO     | veena3.local |   Health: http://0.0.0.0:8000/v1/tts/health
2026-02-17 01:17:10 | INFO     | veena3.local |   Generate: POST http://0.0.0.0:8000/v1/tts/generate
2026-02-17 01:17:10 | INFO     | veena3.local |   Metrics: http://0.0.0.0:8000/v1/tts/metrics
2026-02-17 01:17:10 | INFO     | veena3.local |   Docs: http://0.0.0.0:8000/docs
2026-02-17 01:17:10 | INFO     | veena3.local | ============================================================
✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cpu
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
🚀 SparkTTSPipeline initialized
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)
{"timestamp": "2026-02-17T01:17:10.631994+00:00", "level": "INFO", "logger": "veena3modal.api.websocket_handler", "message": "WebSocket route /v1/tts/ws registered"}
{"timestamp": "2026-02-17T01:17:10.632137+00:00", "level": "INFO", "logger": "veena3modal.api.fastapi_app", "message": "WebSocket support enabled at /v1/tts/ws"}
2026-02-17 01:17:10 | INFO     | veena3modal.api.websocket_handler | WebSocket route /v1/tts/ws registered
2026-02-17 01:17:10 | INFO     | veena3modal.api.fastapi_app | WebSocket support enabled at /v1/tts/ws
INFO:     Started server process [3732972]
INFO:     Waiting for application startup.
INFO:     Application startup complete.
INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)


curl -sS -m 120 -o /tmp/tts_one.wav -D - -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": false, "max_tokens": 128}'


Output:
Chunk ID: 0a8315
Wall time: 30.0009 seconds
Process running with session ID 16403
Original token count: 0
Output:


Output:
Chunk ID: eb6213
Wall time: 30.0013 seconds
Process running with session ID 16403
Original token count: 0
Output:


Output:
Chunk ID: 3ca1b3
Wall time: 30.0014 seconds
Process running with session ID 16403
Original token count: 0
Output:


Output:
Chunk ID: e030ef
Wall time: 5.0010 seconds
Process running with session ID 96712
Original token count: 134
Output:
{"timestamp": "2026-02-17T01:17:38.100059+00:00", "level": "INFO", "logger": "veena3modal.tts", "message": "request_received", "request_id": "369b558c-950c-4494-85bd-d5c17f2320d1", "event": "request_received", "text_length": 5, "speaker": "lipakshi", "stream": false, "format": "wav"}
2026-02-17 01:17:38 | INFO     | veena3modal.tts | request_received
2026-02-17 01:17:38 | INFO     | veena3modal.services.tts_runtime | tts_runtime_nonstream_start
2026-02-17 01:17:38 | INFO     | veena3modal.core.pipeline | tts_llm_generate_start


Output:
Chunk ID: 0bbeac
Wall time: 19.5639 seconds
Process exited with code 28
Original token count: 20
Output:
curl: (28) Operation timed out after 120001 milliseconds with 0 bytes received


Output:
Chunk ID: 7ba9bc
Wall time: 5.0008 seconds
Process running with session ID 96712
Original token count: 0
Output:


curl -sS http://localhost:8000/v1/tts/metrics | rg 'requests_completed_total|requests_failed_total'


Output:
Chunk ID: 2d96f7
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 63
Output:
# HELP veena3_tts_requests_completed_total Total TTS requests completed successfully
# TYPE veena3_tts_requests_completed_total counter
# HELP veena3_tts_requests_failed_total Total TTS requests failed
# TYPE veena3_tts_requests_failed_total counter


ps -p 3732972,3733081 -o pid,pcpu,pmem,etime,cmd && nvidia-smi --query-compute-apps=pid,name,used_gpu_memory --format=csv,noheader,nounits


Output:
Chunk ID: a65799
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 78
Output:
    PID %CPU %MEM     ELAPSED CMD
3732972 38.1  1.4       03:16 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.5 --log-level INFO --port 8000
3733081 17.1  1.7       03:09 VLLM::EngineCore
3733081, VLLM::EngineCore, 41442
3732972, python3, 414


Output:
Chunk ID: 9cd93f
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 268
Output:
scripts/detailed_word_analysis.py
scripts/measure_ttfb_detailed_v2.py
scripts/analyze_missing_words.py
scripts/create_visual_map.py
scripts/pattern_analysis.py
scripts/validate_true_streaming.py
scripts/measure_client_ttfb.py
scripts/quick_asr_check.py
scripts/stress_test_production.py
scripts/asr_with_openai_final.py
scripts/validation/final_validation.py
scripts/elevenlabs_verify_fix.py
scripts/validate_indic_normalization.py
scripts/validate_text_normalization.py
scripts/test_chunking_no_emotions.py
scripts/test_true_streaming.py
scripts/elevenlabs_asr_analysis.py
scripts/validate_modal_endpoint.py
scripts/asr_validation_openai.py
scripts/wer_test_with_emotions.py
scripts/test_fix.py
scripts/profile_pipeline.py
scripts/final_asr_verification.py
scripts/test_all_features.py
scripts/setup_local.sh
scripts/comprehensive_validation.py
scripts/stress_test_local.py
scripts/test_correct_format.py
scripts/test_original_bug.py
scripts/final_chunking_test.sh
scripts/validate_chunking_asr.py
scripts/stress_test_optimized_local.py
scripts/measure_ttfb_detailed.py


sed -n '1,320p' scripts/stress_test_local.py


Output:
Chunk ID: 893948
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 2590
Output:
#!/usr/bin/env python3
"""
Local Stress Test for Veena3 TTS Server.

Progressive load ramp with GPU monitoring, latency percentiles,
error categorization, and throughput measurement.

Unlike the Modal load tests, this runs locally with:
- No network latency (localhost)
- Direct GPU monitoring (nvidia-smi + torch.cuda)
- Higher concurrency ceiling (single-machine, no rate limits)
- Real streaming TTFB measurement (byte-level timing)

Usage:
    # Full suite (ramps 1 → 5 → 10 → 20 → 50 → 100 concurrent)
    python scripts/stress_test_local.py

    # Quick smoke test (1 → 5 → 10 only)
    python scripts/stress_test_local.py --quick

    # Custom ramp
    python scripts/stress_test_local.py --levels 1,5,10,20

    # Streaming only
    python scripts/stress_test_local.py --stream-only

    # Non-streaming only
    python scripts/stress_test_local.py --no-stream

    # Custom endpoint
    python scripts/stress_test_local.py --url http://localhost:8080
"""

import argparse
import asyncio
import json
import os
import statistics
import subprocess
import sys
import time
import threading
from dataclasses import dataclass, field
from typing import List, Optional, Dict, Any, Tuple

import httpx

# === Configuration ===

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

# Test texts at different lengths to stress different parts of the pipeline
TEST_TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This sentence tests basic text normalization and generation quality "
        "with moderate length input that should complete in reasonable time."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast long shadows "
        "over crowded streets, there lived an old bookkeeper named Margaret. She had spent "
        "forty years cataloging the stories of others, yet never found time to write her own. "
        "One rainy Tuesday afternoon, she discovered a leather-bound journal tucked behind "
        "a shelf of dusty encyclopedias. Its pages were blank, waiting. She picked up her pen "
        "and began to write. The words flowed like water from a broken dam."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

# === GPU Monitoring ===

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    """Background thread that polls nvidia-smi for GPU stats."""

    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=5)

    def _poll_loop(self):
        while self._running:
            try:
                result = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True, text=True, timeout=5,
                )
                if result.returncode == 0:
                    parts = result.stdout.strip().split(", ")
                    if len(parts) >= 4:
                        self.snapshots.append(GPUSnapshot(
                            timestamp=time.time(),
                            memory_used_mb=float(parts[0]),
                            memory_total_mb=float(parts[1]),
                            gpu_utilization_pct=float(parts[2]),
                            temperature_c=float(parts[3]),
                        ))
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {"error": "no GPU data collected"}

        mem_used = [s.memory_used_mb for s in self.snapshots]
        gpu_util = [s.gpu_utilization_pct for s in self.snapshots]
        temps = [s.temperature_c for s in self.snapshots]

        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem_used),
                "max": max(mem_used),
                "avg": statistics.mean(mem_used),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(gpu_util),
                "max": max(gpu_util),
                "avg": statistics.mean(gpu_util),
            },
            "temperature_c": {
                "min": min(temps),
                "max": max(temps),
                "avg": statistics.mean(temps),
            },
        }

# === Request Result ===

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float  # Total wall-clock time
    ttfb_ms: float = 0.0  # Time to first byte (from HTTP response start)
    server_ttfb_ms: float = 0.0  # Server-reported TTFB (X-TTFB-ms header)
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    error: Optional[str] = None
    speaker: str = ""
    text_length: int = 0
    stream: bool = False

# === Load Level Result ===

@dataclass
class LevelResult:
    level_name: str
    concurrency: int
    total_requests: int
    mode: str  # "non-streaming" or "streaming"
    text_category: str  # "short", "medium", "long", "mixed"
    results: List[RequestResult] = field(default_factory=list)
    wall_time_seconds: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_seconds if self.wall_time_seconds > 0 else 0

    def _percentile(self, values: List[float], pct: float) -> float:
        if not values:
            return 0.0
        s = sorted(values)
        idx = int(len(s) * pct / 100)
        return s[min(idx, len(s) - 1)]

    @property
    def latency_stats(self) -> Dict[str, float]:
        lats = [r.latency_ms for r in self.successes]
        if not lats:
            return {}
        return {
            "min": min(lats),
            "avg": statistics.mean(lats),
            "p50": self._percentile(lats, 50),
            "p95": self._percentile(lats, 95),
            "p99": self._percentile(lats, 99),
            "max": max(lats),
        }

    @property
    def ttfb_stats(self) -> Dict[str, float]:
        ttfbs = [r.ttfb_ms for r in self.successes if r.ttfb_ms > 0]
        if not ttfbs:
            return {}
        return {
            "min": min(ttfbs),
            "avg": statistics.mean(ttfbs),
            "p50": self._percentile(ttfbs, 50),
            "p95": self._percentile(ttfbs, 95),
            "max": max(ttfbs),
        }

    @property
    def audio_duration_total(self) -> float:
        return sum(r.audio_seconds for r in self.successes)

    @property
    def effective_rtf(self) -> float:
        """Wall time / total audio produced. Lower = better."""
        total_audio = self.audio_duration_total
        return self.wall_time_seconds / total_audio if total_audio > 0 else 0

    def error_summary(self) -> Dict[str, int]:
        errors: Dict[str, int] = {}
        for r in self.failures:
            key = r.error or f"HTTP {r.status_code}"
            errors[key[:80]] = errors.get(key[:80], 0) + 1
        return errors

# === HTTP Client ===

async def make_request(
    client: httpx.AsyncClient,
    base_url: str,
    text: str,
    speaker: str,
    stream: bool = False,
    timeout: float = 120.0,
) -> RequestResult:
    """Make a single TTS request with timing."""
    url = f"{base_url}{GENERATE_PATH}"
    payload = {
        "text": text,
        "speaker": speaker,
        "stream": stream,
        "format": "wav",
    }

    start = time.time()
    ttfb_time = 0.0

    try:
        if stream:
            # Streaming: measure true TTFB (time to first byte of audio)
            async with client.stream(
                "POST", url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            ) as response:
                first_byte = True
                total_bytes = 0
                async for chunk in response.aiter_bytes():
                    if first_byte:
                        ttfb_time = (time.time() - start) * 1000
                        first_byte = False
                    total_bytes += len(chunk)

                latency = (time.time() - start) * 1000

                if response.status_code == 200:
                    # Estimate audio duration from bytes (16kHz, 16-bit, mono, minus WAV header)
                    pcm_bytes = max(0, total_bytes - 44)
                    audio_seconds = pcm_bytes / (16000 * 2)

                    return RequestResult(
                        success=True,
                        status_code=200,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        audio_bytes=total_bytes,
                        audio_seconds=audio_seconds,
                        speaker=speaker,
                        text_length=len(text),
                        stream=True,
                    )
                else:
                    return RequestResult(
                        success=False,
                        status_code=response.status_code,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        error=f"HTTP {response.status_code}",
                        speaker=speaker,


python3 scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 128 --timeout-seconds 5 --no-chunking --output /tmp/bench_short_fail_t5.json


Output:
Chunk ID: a7e569
Wall time: 30.0007 seconds
Process running with session ID 66483
Original token count: 0
Output:


Output:
Chunk ID: a8d45f
Wall time: 30.0007 seconds
Process running with session ID 66483
Original token count: 0
Output:


pgrep -af 'scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 128'


Output:
Chunk ID: f42f34
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 44
Output:
3734153 python3 scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 128 --timeout-seconds 5 --no-chunking --output /tmp/bench_short_fail_t5.json


I’ve got a benchmark run in progress with short request-level timeouts to force deterministic outcomes. It’s still executing (about the expected 1/2/4-level sweep with 5s per-call timeout), so I’ll let it complete and then extract per-request timing/inflight data.


Output:
Chunk ID: 3567f5
Wall time: 30.0006 seconds
Process running with session ID 66483
Original token count: 0
Output:


Output:
Chunk ID: 5d57fe
Wall time: 3.4970 seconds
Process exited with code 0
Original token count: 621
Output:
  Health: healthy
  Version: LLM
  GPU: True

Warming up (1 request, non-streaming)...
  warmup ok: 0%

========================================================================
  1C-SHORT-NON-STREAMING
  10 requests, 1 concurrent, non-streaming, chunking=False
========================================================================
  Success:     0/10 (0%)
  Wall time:   50.13s
  Throughput:  0.20 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   5% avg, 74% peak
    Temp:   30C avg, 32C peak

  Errors:
    [10x] TIMEOUT
  cooldown 3s...

========================================================================
  2C-SHORT-NON-STREAMING
  10 requests, 2 concurrent, non-streaming, chunking=False
========================================================================
  Success:     0/10 (0%)
  Wall time:   25.09s
  Throughput:  0.40 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   7% avg, 73% peak
    Temp:   30C avg, 32C peak

  Errors:
    [10x] TIMEOUT
  cooldown 3s...

========================================================================
  4C-SHORT-NON-STREAMING
  12 requests, 4 concurrent, non-streaming, chunking=False
========================================================================
  Success:     0/12 (0%)
  Wall time:   15.08s
  Throughput:  0.80 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   9% avg, 76% peak
    Temp:   31C avg, 32C peak

  Errors:
    [12x] TIMEOUT

==============================================================================================================
  STRESS BENCHMARK SUMMARY
==============================================================================================================
Level                        OK%     RPS     p50     p95  GPU%   GPUmem    RTF Inflight
--------------------------------------------------------------------------------------------------------------
1c-short-non-streaming       0%    0.20     ---     ---    5%   41871MB  0.000        0
2c-short-non-streaming       0%    0.40     ---     ---    7%   41871MB  0.000        0
4c-short-non-streaming       0%    0.80     ---     ---    9%   41871MB  0.000        0
==============================================================================================================

Results written to /tmp/bench_short_fail_t5.json


python3 - <<'PY'
import json
p='/tmp/bench_short_fail_t5.json'
with open(p) as f:
    data=json.load(f)
print('levels', len(data))
for lvl in data:
    s=lvl['success_rate'];
    print(lvl['level'], len(lvl['requests']), 'succ', int(s*100))
    # print one request example
    print('errors', lvl['errors'])
    if lvl['requests']:
        rr=lvl['requests'][0]
        print('sample timing keys', sorted(rr['timing'].keys())[:10], '...')
    print()
PY


Output:
Chunk ID: d1086d
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 65
Output:
levels 3
1c-short-non-streaming 10 succ 0
errors {'TIMEOUT': 10}
sample timing keys [] ...

2c-short-non-streaming 10 succ 0
errors {'TIMEOUT': 10}
sample timing keys [] ...

4c-short-non-streaming 12 succ 0
errors {'TIMEOUT': 12}
sample timing keys [] ...


python3 - <<'PY'
import json
p='/tmp/bench_short_fail_t5.json'
with open(p) as f:
    data=json.load(f)
for lvl in data:
    print('===', lvl['level'])
    for req in lvl['requests'][:2]:
        keys={k:v for k,v in req.items() if k in ('success','status_code','latency_ms','error','request_id','ttfb_ms','server_ttfb_ms','request_inflight','audio_bytes')}
        print(keys)
    print('---')
PY


Output:
Chunk ID: b04640
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 310
Output:
=== 1c-short-non-streaming
{'request_id': '', 'success': False, 'status_code': 0, 'latency_ms': 5006.50691986084, 'ttfb_ms': 0.0, 'server_ttfb_ms': 0.0, 'audio_bytes': 0, 'request_inflight': None, 'error': 'TIMEOUT'}
{'request_id': '', 'success': False, 'status_code': 0, 'latency_ms': 5008.047103881836, 'ttfb_ms': 0.0, 'server_ttfb_ms': 0.0, 'audio_bytes': 0, 'request_inflight': None, 'error': 'TIMEOUT'}
---
=== 2c-short-non-streaming
{'request_id': '', 'success': False, 'status_code': 0, 'latency_ms': 5009.214162826538, 'ttfb_ms': 0.0, 'server_ttfb_ms': 0.0, 'audio_bytes': 0, 'request_inflight': None, 'error': 'TIMEOUT'}
{'request_id': '', 'success': False, 'status_code': 0, 'latency_ms': 5008.847236633301, 'ttfb_ms': 0.0, 'server_ttfb_ms': 0.0, 'audio_bytes': 0, 'request_inflight': None, 'error': 'TIMEOUT'}
---
=== 4c-short-non-streaming
{'request_id': '', 'success': False, 'status_code': 0, 'latency_ms': 5014.258146286011, 'ttfb_ms': 0.0, 'server_ttfb_ms': 0.0, 'audio_bytes': 0, 'request_inflight': None, 'error': 'TIMEOUT'}
{'request_id': '', 'success': False, 'status_code': 0, 'latency_ms': 5013.864517211914, 'ttfb_ms': 0.0, 'server_ttfb_ms': 0.0, 'audio_bytes': 0, 'request_inflight': None, 'error': 'TIMEOUT'}
---


Summary: auto


sed -n '1,220p' veena3modal/core/constants.py


Output:
Chunk ID: 22d771
Wall time: 0.0502 seconds
Process exited with code 0
Original token count: 1489
Output:
"""
Spark TTS Constants

Token IDs, special tokens, speakers, and emotions for Spark TTS model.
Migrated from Veena3/Orpheus to Spark TTS (BiCodec) architecture.
"""

import os

# Stop tokens for generation
# Spark TTS uses <|im_end|> as stop token
TRAINING_STOP_TOKEN_IDS = ["<|im_end|>"]  # String-based stop token for Spark TTS

# Spark TTS Speaker System - 12 Speakers from Training
# Maps user-facing speaker names to internal speaker IDs (speaker_0 to speaker_11)
# Model: BayAreaBoys/spark_tts_4speaker (HuggingFace) has 12 speaker tokens
SPEAKER_MAP = {
    "lipakshi": 0,    # speaker_0
    "vardan": 1,      # speaker_1
    "reet": 2,        # speaker_2
    "Nandini": 3,     # speaker_3
    "krishna": 4,     # speaker_4
    "anika": 5,       # speaker_5
    "adarsh": 6,      # speaker_6
    "Nilay": 7,       # speaker_7
    "Aarvi": 8,       # speaker_8
    "Asha": 9,        # speaker_9
    "Bittu": 10,      # speaker_10
    "Mira": 11,       # speaker_11
}

# Friendly speaker name mappings (user-facing → internal)
# Users can use these friendly names instead of internal names
FRIENDLY_SPEAKER_MAP = {
    "Mitra": "lipakshi",
    "Aaranya": "reet",
    "Taru": "Nandini",
    "Neer": "Nilay",
    "Dhruva": "vardan",
    "Ira": "anika",
    "Veda": "adarsh",
    "Aria": "krishna",
    # Direct mappings for new speakers
    "Aarvi": "Aarvi",
    "Asha": "Asha",
    "Bittu": "Bittu",
    "Mira": "Mira",
}

# Valid speaker names (for API validation)
# Include both internal names and friendly names
INDIC_SPEAKERS = list(SPEAKER_MAP.keys())
ALL_SPEAKER_NAMES = INDIC_SPEAKERS + list(FRIENDLY_SPEAKER_MAP.keys())

def resolve_speaker_name(name: str) -> str:
    """
    Resolve friendly speaker name to internal name.

    Args:
        name: Speaker name (friendly or internal)

    Returns:
        Internal speaker name

    Examples:
        resolve_speaker_name("Mitra") -> "lipakshi"
        resolve_speaker_name("lipakshi") -> "lipakshi"
    """
    # Check if it's a friendly name
    if name in FRIENDLY_SPEAKER_MAP:
        return FRIENDLY_SPEAKER_MAP[name]

    # Check if it's already an internal name
    if name in INDIC_SPEAKERS:
        return name

    # Invalid speaker name
    raise ValueError(f"Invalid speaker name: {name}. Valid names: {', '.join(ALL_SPEAKER_NAMES)}")

# Spark TTS Emotion Tags (bracket format: [emotion])
# NOTE: API users must now use [emotion] instead of <emotion>
INDIC_EMOTION_TAGS = [
    "[angry]",
    "[curious]",
    "[excited]",
    "[giggle]",
    "[laughs harder]",
    "[laughs]",
    "[screams]",
    "[sighs]",
    "[sings]",
    "[whispers]"
]

# Legacy emotion tags for backward compatibility mapping
# Maps old <emotion> format to new [emotion] format
LEGACY_EMOTION_MAP = {
    "<angry>": "[angry]",
    "<curious>": "[curious]",
    "<excited>": "[excited]",
    "<giggle>": "[giggle]",
    "<laugh_harder>": "[laughs harder]",
    "<laugh>": "[laughs]",
    "<scream>": "[screams]",
    "<sigh>": "[sighs]",
    "<sing>": "[sings]",
    "<whisper>": "[whispers]"
}

# All emotion tags (for validation)
ALL_EMOTION_TAGS = INDIC_EMOTION_TAGS

# Model configuration
DEFAULT_MODEL_PATH = "BayAreaBoys/spark_tts_4speaker"
DEFAULT_MAX_MODEL_LEN = 4096  # Lower than old model (was 8192)

# BiCodec Audio Tokenizer
# Location: resolved from env var MODEL_PATH or default
BICODEC_TOKENIZER_PATH = os.environ.get('MODEL_PATH', os.environ.get('SPARK_TTS_MODEL_PATH', '/models/spark_tts_4speaker'))

# Audio configuration
AUDIO_SAMPLE_RATE = 16000  # Changed from 24kHz to 16kHz for BiCodec
AUDIO_CHANNELS = 1
AUDIO_BITS_PER_SAMPLE = 16

# Generation defaults for Spark TTS
DEFAULT_TEMPERATURE = 0.8  # Higher than old model (was 0.4)
DEFAULT_TOP_K = 50
DEFAULT_TOP_P = 1.0
DEFAULT_MAX_TOKENS = 4096  # Increased to handle complex multilingual generation
DEFAULT_MIN_TOKENS = 28  # Minimum tokens for generation (legacy compatibility)
DEFAULT_REPETITION_PENALTY = 1.0  # Repetition penalty (legacy compatibility)
DEFAULT_SEED = None  # None = random, set integer for reproducibility

# vLLM Configuration optimizations for Spark TTS
# NOTE: gpu_memory_utilization adjusted based on available memory
# Production: 0.85, Limited memory: 0.3-0.5
#
# OPTIMIZATION Dec 2025:
# - enable_chunked_prefill: Prevents long prompts from blocking concurrent streams
# - async_scheduling: DISABLED - caused EngineCore issues in production
# OPTIMIZATION Feb 2026:
# Reduced gpu_memory_utilization from 0.85 to 0.25
# Model is only 0.5B (~1.3GB weights). At 0.85, vLLM pre-allocated 65GB for KV cache
# (enough for 1,399 concurrent seqs) when peak actual need is ~100 concurrent.
# At 0.25: ~18GB KV cache, supports ~380 concurrent seqs (4x peak), frees ~48GB VRAM.
# This enables fitting on L4 (24GB) or even T4 (16GB) GPUs.
VLLM_CONFIG = {
    "dtype": "bfloat16",
    "gpu_memory_utilization": 0.25,  # 0.5B model needs <2GB; 0.25 gives ~380 concurrent seqs
    "max_model_len": 4096,
    "max_num_batched_tokens": 4096,
    "enable_prefix_caching": True,  # Cache common prompts for faster TTFB
    "enable_chunked_prefill": True,  # Chunk long prefills to avoid blocking concurrent streams
    "enforce_eager": False,  # Enable CUDA graphs for low latency
    "disable_log_stats": False,
    "trust_remote_code": True,
    "tensor_parallel_size": 1,
}

# Streaming configuration (may need adjustment for BiCodec)
STREAM_BUFFER_SIZE = 50  # Buffer size for streaming (BiCodec-specific, TBD)
BICODEC_BATCH_SIZE = 64
BICODEC_BATCH_TIMEOUT_MS = 15

# Backward compatibility: Old SNAC constants for files not yet migrated
# These are NOT used by Spark TTS but needed by legacy streaming code
# TODO: Remove once streaming_pipeline.py is fully migrated to BiCodec
CODE_START_TOKEN_ID = 128257  # Legacy SNAC constant
CODE_END_TOKEN_ID = 128258    # Legacy SNAC constant
SNAC_MIN_ID = 128266          # Legacy SNAC constant
SNAC_MAX_ID = 156937          # Legacy SNAC constant


To validate whether this is a generation-path stall versus API framing, I’m doing a low-level generation probe directly through the loaded pipeline (prompt → vLLM iterator) to see whether the iterator yields any first batch at all.


python3 - <<'PY'
import asyncio, os
from vllm import SamplingParams
from veena3modal.core.constants import TRAINING_STOP_TOKEN_IDS
from veena3modal.services.tts_runtime import initialize_runtime

os.environ['NUMBA_DISABLE_JIT']='1'
os.environ['BICODEC_DEVICE']='cpu'
os.environ['GPU_MEMORY_UTILIZATION']='0.5'

async def main():
    runtime = initialize_runtime(
        model_path='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM',
        bicodec_path='/home/ubuntu/veenaModal/models/spark_tts_4speaker',
        device='cuda',
        gpu_memory_utilization=0.5,
        enable_sr=False,
    )
    prompt = runtime.prompt_builder.build_prefix('lipakshi', 'Hello')
    params = SamplingParams(
        temperature=0.8,
        top_k=50,
        top_p=1.0,
        max_tokens=128,
        repetition_penalty=1.05,
        stop=TRAINING_STOP_TOKEN_IDS,
        skip_special_tokens=False,
    )
    agen = runtime.model.engine.generate(prompt=prompt, sampling_params=params, request_id='probe-debug')
    try:
        first = await asyncio.wait_for(agen.__anext__(), timeout=20)
        print('first output received', bool(first), 'has outputs', hasattr(first, 'outputs'))
        print('token len', len(first.outputs[0].token_ids), 'text len', len(first.outputs[0].text))
    except Exception as e:
        import traceback; traceback.print_exc()

asyncio.run(main())
PY