Output:
Chunk ID: d17a21
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 4419
Output:
"""
Veena3 Streaming Pipeline - Sliding Window Approach

Implements streaming for both SNAC and BiCodec tokens.

For BiCodec (Spark TTS):
1. Generate tokens from vLLM
2. Parse NEW token IDs incrementally (O(1) per token via cache)
3. Buffer semantic + global tokens
4. Apply sliding window (every N token pairs)
5. Decode and stream audio chunks

OPTIMIZATION (Dec 2025):
- Replaced O(n²) pattern (decode-all + regex-all per iteration)
- Now uses incremental token parsing with BiCodecTokenParser
- ~10x CPU reduction in streaming hot loop

For SNAC (legacy):
1. Filter SNAC token IDs directly
2. Apply sliding window
3. Decode and stream
"""

import asyncio
import os
import re
import logging
import torch
from typing import AsyncGenerator, Optional, List, Dict, Any
from vllm import SamplingParams
from veena3modal.audio.crossfade import crossfade_bytes_int16
from veena3modal.core.token_utils import BiCodecTokenParser

from veena3modal.core.constants import (
    CODE_END_TOKEN_ID,
    CODE_START_TOKEN_ID,
    SNAC_MIN_ID,
    SNAC_MAX_ID,
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_MIN_TOKENS,
    DEFAULT_REPETITION_PENALTY,
    DEFAULT_SEED,
)

logger = logging.getLogger(__name__)

def _detailed_timing_enabled() -> bool:
    """Enable per-batch token/decode timing logs."""
    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}

class Veena3SlidingWindowPipeline:
    """
    Streaming TTS pipeline using sliding window approach.

    This eliminates choppy audio and popping artifacts by:
    - Decoding overlapping 28-token windows (4 frames)
    - Keeping only the middle 2048 samples from each decode
    - Creating natural continuity between chunks

    Based on the official Canopy Labs implementation.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        snac_decoder,
    ):
        """
        Initialize sliding window streaming pipeline.

        Args:
            model: Veena3Model instance
            prompt_builder: Veena3PromptBuilder instance
            snac_decoder: SNACDecoder instance (with batching enabled)
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.snac_decoder = snac_decoder

        # OPTIMIZATION: Pre-warm BiCodecTokenParser once at init, not per-request
        # Saves ~123ms per streaming TTFB by avoiding repeated 166K-entry vocab iteration
        tokenizer = getattr(model, "tokenizer", None)
        if tokenizer is None:
            tokenizer = getattr(model.engine, "tokenizer", None)
        self.token_parser = BiCodecTokenParser(tokenizer) if tokenizer else None

        print(f"🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)")

    async def generate_speech_stream(
        self,
        description: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech audio with sliding window streaming.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            description: Character/voice description
            text: Text to synthesize (with optional <emotion> tags)
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops

        Yields:
            Audio bytes (int16 PCM, 24kHz mono)
        """
        print(f"\n🌊 Sliding window streaming generation")
        print(f"📝 Description: {description[:80]}...")
        print(f"💬 Text: {text}")

        # Build prompt
        prompt = self.prompt_builder.build_prefix(description, text)
        print(f"✅ Prompt built ({len(prompt)} chars)")

        # Configure sampling
        sampling_params = SamplingParams(
            temperature=temperature,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop_token_ids=[CODE_END_TOKEN_ID],  # Only audio EOS
            seed=seed,  # None = random, int = reproducible
        )

        print(f"🎲 Sampling: temp={temperature}, top_p={top_p}, sliding_window=28 tokens")

        # Token buffer - keeps ALL tokens (not chunked)
        token_buffer = []
        total_tokens_generated = 0
        total_audio_chunks = 0

        # Generate tokens with vLLM (streaming)
        print(f"🔮 Starting token generation...")

        # Generate unique request ID for concurrent streaming support
        import uuid
        import time
        request_id = f"slide-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Stream tokens with sliding window
        async for request_output in results_generator:
            # Extract generated token IDs
            generated_ids = request_output.outputs[0].token_ids

            # Process only new tokens
            new_tokens = generated_ids[total_tokens_generated:]
            total_tokens_generated = len(generated_ids)

            # Filter and buffer SNAC tokens
            for token_id in new_tokens:
                if SNAC_MIN_ID <= token_id <= SNAC_MAX_ID:
                    token_buffer.append(token_id)

                    # Process every 7 tokens (1 frame) when we have enough for sliding window
                    # Official approach: once count > 27, take last 28 tokens
                    if len(token_buffer) % 7 == 0 and len(token_buffer) > 27:
                        # Sliding window: take last 28 tokens (4 frames)
                        window_tokens = token_buffer[-28:]

                        # Decode with sliding window mode (returns middle 2048 samples only)
                        if self.snac_decoder.enable_batching:
                            audio_bytes = await self.snac_decoder.decode_single_async(
                                window_tokens,
                                trim_warmup=False,  # Sliding window handles trimming
                                use_sliding_window=True  # CRITICAL: Use sliding window mode
                            )
                        else:
                            audio_bytes = self.snac_decoder.decode_to_bytes(
                                window_tokens,
                                trim_warmup=False,
                                use_sliding_window=True
                            )

                        if audio_bytes:
                            total_audio_chunks += 1
                            if total_audio_chunks == 1:
                                print(f"🎵 First chunk decoded ({len(audio_bytes)} bytes, sliding window)")
                            yield audio_bytes

        # Note: No final chunk processing needed - sliding window handles all tokens
        # as they come in (every 7 tokens after the first 28)

        print(f"✅ Sliding window streaming complete: {total_tokens_generated} tokens → {total_audio_chunks} audio chunks")

    def _extract_bicodec_tokens_from_text(self, text: str) -> tuple[List[int], List[int]]:
        """
        Extract BiCodec semantic and global tokens from generated text.

        Args:
            text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
        """
        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", text)
        semantic_ids = [int(t) for t in semantic_matches]

        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", text)
        global_ids = [int(t) for t in global_matches]

        return semantic_ids, global_ids

    async def generate_speech_stream_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,  # Added for Spark TTS parity with non-streaming
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        request_id: Optional[str] = None,
        request_metrics: Optional[Dict[str, Any]] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech audio with sliding window streaming for Indic model.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "<laugh> Hello there!"
                - "नमस्ते! <excited> आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops
            seed: Random seed for reproducibility

        Yields:
            Audio bytes (int16 PCM, 16kHz mono - BiCodec)
        """
        import statistics
        import time
        t_start = time.time()
        emit_timing = _detailed_timing_enabled()
        request_metrics = request_metrics or {}
        request_metrics.update(
            {
                "request_id": request_id,
                "speaker": speaker,
                "text_length": len(text),
            }
        )

        # Build prompt using Indic prompt builder
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling (matching non-streaming pipeline parameters)
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID (128258)
        # which doesn't exist in the Spark vocab and would cause generation to run to max_tokens
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,  # Keep BiCodec tokens in output
            seed=seed,
        )

        # BiCodec token buffers (separate for semantic and global)
        semantic_buffer = []
        global_buffer = []
        processed_token_count = 0  # Track how many vLLM tokens we've parsed
        total_audio_chunks = 0

        # BiCodec streaming configuration
        EXPECTED_GLOBAL_COUNT = 32  # BiCodec uses 32 fixed global tokens (per paper)
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8, -120ms TTFB)
        # OPTIMIZATION: Increased from 24 to 48 to reduce O(n) re-decode calls by 2x.
        # At 50 TPS, 48 tokens = ~960ms audio per chunk. Cuts total decode calls roughly in half.
        DECODE_INTERVAL = 48
        CROSSFADE_MS = 50
        # OPTIMIZATION: Windowed decode - only decode last WINDOW_SIZE tokens instead of ALL.
        # BiCodec WaveGenerator has finite receptive field (~64 tokens based on kernel sizes).
        # Window must be >= receptive_field + DECODE_INTERVAL to avoid boundary artifacts.
        WINDOW_SIZE = 128  # 128 tokens covers receptive field + margin for crossfade

        # Generate tokens with vLLM (streaming)
        if request_id is None:
            import uuid
            request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
            request_metrics["request_id"] = request_id

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser
        if token_parser is None:
            raise RuntimeError("BiCodecTokenParser unavailable for streaming")

        t_gen_start = time.time()
        batch_wall_ms: List[float] = []
        batch_token_counts: List[int] = []
        batch_gpu_ms: List[float] = []
        parse_wall_ms_list: List[float] = []
        decode_calls = 0
        decode_wall_ms = 0.0
        decode_gpu_ms = 0.0
        decode_wall_samples: List[float] = []
        decode_gpu_samples: List[float] = []

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track timing
        t_first_token = None
        t_first_chunk_ready = None
        t_first_chunk_decoded = None
        iteration_count = 0
        last_decode_count = 0

        # Track audio samples EMITTED to the user (not just decoded)
        # BiCodec produces ~320 samples per semantic token at 16kHz
        # CRITICAL: This tracks samples that have been YIELDED, not held in tail
        total_samples_emitted_to_user = 0
        # Keep a tail to enable crossfading between consecutive emitted chunks
        previous_chunk_tail: Optional[bytes] = None

        # Stream tokens with BiCodec extraction
        async for request_output in results_generator:
            iteration_count += 1
            t_iter = time.time()

            # Track first token received
            if t_first_token is None:
                t_first_token = t_iter

            # Extract generated token IDs
            generated_ids = request_output.outputs[0].token_ids

            parse_start = time.time()
            # OPTIMIZATION: Only process NEW tokens (incremental parsing)
            # Old code: tokenizer.decode(ALL) + re.findall(ALL) = O(n²) per stream
            # New code: parse only new tokens = O(1) per token
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)

            # Parse new tokens and append to buffers (in-place modification)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
            parse_wall_ms = (time.time() - parse_start) * 1000
            parse_wall_ms_list.append(parse_wall_ms)
            batch_token_counts.append(len(new_token_ids))
            batch_wall_ms_value = (time.time() - t_iter) * 1000 if t_iter else 0.0
            batch_wall_ms.append(batch_wall_ms_value)
            if emit_timing:
                logger.info(
                    "tts_stream_llm_batch",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "batch_index": len(batch_wall_ms),
                        "batch_tokens": len(new_token_ids),
                        "llm_batch_wall_ms": int(batch_wall_ms_value),
                        "llm_batch_gpu_ms": int(batch_gpu_ms[-1]) if batch_gpu_ms else None,
                    },
                )

            # TRUE STREAMING with BiCodec two-phase generation:
            # Phase 1: Generate 32 global tokens (pre-roll)
            # Phase 2: Stream semantic tokens incrementally

            # Check if we have the 32 global tokens (pre-roll complete)
            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                # Pre-roll complete! Now we can stream semantic tokens
                if t_first_chunk_ready is None and len(semantic_buffer) >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    t_first_chunk_ready = time.time()

                # Stream semantic tokens incrementally
                # Only decode if we have enough NEW semantic tokens since last decode
                semantic_count = len(semantic_buffer)
                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    # Check if we should decode (every DECODE_INTERVAL new semantic tokens)
                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
                        last_decode_count = semantic_count

                        # OPTIMIZATION: Windowed decode instead of full re-decode
                        # Only decode the last WINDOW_SIZE tokens (O(1) per chunk instead of O(n))
                        # For first few chunks when total tokens < WINDOW_SIZE, decode all
                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]

                        if len(semantic_buffer) <= WINDOW_SIZE:
                            # Early phase: decode all tokens (window not yet relevant)
                            decode_semantic = semantic_buffer
                        else:
                            # Windowed: only decode last WINDOW_SIZE tokens
                            decode_semantic = semantic_buffer[-WINDOW_SIZE:]

                        decode_gpu_ms_this = None
                        t_decode_start = time.time()
                        if emit_timing and torch.cuda.is_available():


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '420,860p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 3ae79d
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 5366
Output:
                        if emit_timing and torch.cuda.is_available():
                            torch.cuda.synchronize()
                            decode_start_event = torch.cuda.Event(enable_timing=True)
                            decode_end_event = torch.cuda.Event(enable_timing=True)
                            decode_start_event.record()
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=decode_semantic,
                            global_ids=window_global,
                            trim_warmup=False,
                            use_sliding_window=False,
                        )
                        t_decode_end = time.time()
                        decode_calls += 1
                        decode_ms = (t_decode_end - t_decode_start) * 1000
                        decode_wall_ms += decode_ms
                        decode_wall_samples.append(decode_ms)
                        if emit_timing and torch.cuda.is_available():
                            decode_end_event.record()
                            torch.cuda.synchronize()
                            decode_gpu_ms_this = decode_start_event.elapsed_time(decode_end_event)
                            decode_gpu_ms += decode_gpu_ms_this
                            decode_gpu_samples.append(decode_gpu_ms_this)
                            batch_gpu_ms.append(decode_gpu_ms_this)

                        if audio_bytes:
                            total_samples_decoded = len(audio_bytes) // 2

                            if len(semantic_buffer) <= WINDOW_SIZE:
                                # Early phase: extract new samples from full decode
                                if total_samples_decoded > total_samples_emitted_to_user:
                                    new_bytes_start = total_samples_emitted_to_user * 2
                                    new_audio_bytes = audio_bytes[new_bytes_start:]
                                else:
                                    new_audio_bytes = b""
                            else:
                                # Windowed phase: take only the tail portion of the window decode
                                # The last DECODE_INTERVAL tokens' worth of audio is the new content
                                # Each token produces ~320 samples at 16kHz/50TPS
                                new_samples_approx = DECODE_INTERVAL * 320
                                if total_samples_decoded > new_samples_approx:
                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
                                else:
                                    new_bytes_start = 0
                                new_audio_bytes = audio_bytes[new_bytes_start:]

                            if new_audio_bytes:
                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                    previous_chunk_tail,
                                    new_audio_bytes,
                                    sample_rate_hz=16000,
                                    crossfade_ms=CROSSFADE_MS,
                                )

                        samples_emitted_in_this_chunk = len(to_emit) // 2
                        total_samples_emitted_to_user += samples_emitted_in_this_chunk

                        if to_emit:
                            total_audio_chunks += 1
                            if t_first_chunk_decoded is None:
                                t_first_chunk_decoded = t_decode_end

                            if emit_timing:
                                logger.info(
                                    "tts_stream_decode_batch",
                                    extra={
                                        "speaker": speaker,
                                        "request_id": request_id,
                                        "batch_index": decode_calls,
                                        "decode_wall_ms": int((t_decode_end - t_decode_start) * 1000),
                                        "decode_gpu_ms": int(decode_gpu_ms_this)
                                        if decode_gpu_ms_this is not None else None,
                                        "semantic_count": len(semantic_buffer),
                                    },
                                )

                            yield to_emit

        # Final chunk: decode any remaining semantic tokens
        # Use the same 32 global tokens (they get pooled)
        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,  # All semantic tokens
                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],  # Use first 32 global tokens
                use_sliding_window=False,  # Get all samples
                trim_warmup=False
            )
            if audio_bytes:
                # Only yield NEW samples
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    samples_emitted_in_final = len(to_emit) // 2
                    total_samples_emitted_to_user += samples_emitted_in_final

                    if to_emit:
                        total_audio_chunks += 1
                        yield to_emit

        # Flush any remaining tail that was held back for crossfade
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            yield previous_chunk_tail

        # Log completion summary
        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000  # 16kHz sample rate
        ttfb_ms = (
            (t_first_chunk_decoded - t_start) * 1000
            if t_first_chunk_decoded is not None
            else (t_first_token - t_start) * 1000
            if t_first_token is not None
            else 0
        )
        total_wall_ms = (t_end - t_start) * 1000
        request_metrics.update(
            {
                "llm_request_id": request_id,
                "generated_tokens": len(semantic_buffer) + len(global_buffer),
                "semantic_tokens": len(semantic_buffer),
                "global_tokens": len(global_buffer),
                "llm_batch_count": len(batch_wall_ms),
                "llm_batch_tokens": batch_token_counts,
                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_parse_ms": int(sum(parse_wall_ms_list)),
                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
                "decode_calls": decode_calls,
                "llm_decode_wall_ms_total": int(decode_wall_ms),
                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
                "ttfb_ms": int(ttfb_ms),
                "generation_ms": int(total_wall_ms),
                "audio_duration_seconds": audio_duration_s,
                "audio_chunks": total_audio_chunks,
                "samples_emitted": total_samples_emitted_to_user,
                "text_chunked": False,
                "concurrency_mode": "single",
            }
        )
        if emit_timing:
            logger.info("tts_stream_indic_complete", extra=request_metrics)

    async def generate_speech_stream_indic_first_chunk(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        request_id: Optional[str] = None,
        request_metrics: Optional[Dict[str, Any]] = None,
    ) -> AsyncGenerator[tuple, None]:
        """
        Generate speech for the FIRST chunk of multi-chunk text, capturing global tokens.

        This method is specifically for chunked generation: it yields (audio_bytes, global_ids)
        tuples instead of just audio_bytes. The caller captures global_ids from the first
        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.

        Use Case (chunked streaming with voice consistency):
            globals_captured = None
            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
                if globals_captured is None and global_ids:
                    globals_captured = global_ids  # Capture once
                yield audio_bytes
            # Now use globals_captured for subsequent chunks

        Args:
            speaker: Speaker name
            text: First text chunk to synthesize
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max tokens to generate
            repetition_penalty: Prevent repetition
            seed: Random seed for reproducibility

        Yields:
            Tuple of (audio_bytes, global_ids)
            - audio_bytes: Raw PCM audio (int16, 16kHz)
            - global_ids: List of 32 global token IDs (populated after first decode, else empty)

        Thread Safety:
            This method is stateless and thread-safe. Each call creates its own
            request-scoped state. No global state is modified or shared.
        """
        import time
        import torch

        t_start = time.time()
        emit_timing = _detailed_timing_enabled()
        request_metrics = request_metrics or {}
        request_metrics.update(
            {
                "request_id": request_id,
                "speaker": speaker,
                "text_length": len(text),
                "text_chunked": True,
            }
        )

        # Build prompt using Indic prompt builder
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,
            seed=seed,
        )

        # BiCodec token buffers
        semantic_buffer = []
        global_buffer = []
        processed_token_count = 0  # Track how many vLLM tokens we've parsed
        total_audio_chunks = 0

        # BiCodec streaming configuration
        EXPECTED_GLOBAL_COUNT = 32
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10
        DECODE_INTERVAL = 48
        CROSSFADE_MS = 50

        # Generate unique request ID
        import uuid
        if request_id is None:
            request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
            request_metrics["request_id"] = request_id

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser
        if token_parser is None:
            raise RuntimeError("BiCodecTokenParser unavailable for first-chunk streaming")

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        batch_wall_ms: List[float] = []
        batch_gpu_ms: List[float] = []
        batch_token_counts: List[int] = []
        parse_wall_ms_list: List[float] = []
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None
        captured_globals: List[int] = []  # Will be populated once we have 32 globals
        decode_calls = 0
        decode_wall_ms = 0.0
        decode_gpu_ms = 0.0
        decode_wall_samples: List[float] = []
        decode_gpu_samples: List[float] = []
        t_first_token = None
        t_first_chunk_ready = None
        t_first_chunk_decoded = None
        iteration_count = 0

        # Stream tokens with BiCodec extraction
        async for request_output in results_generator:
            iteration_count += 1
            t_iter = time.time()
            if t_first_token is None:
                t_first_token = t_iter

            generated_ids = request_output.outputs[0].token_ids

            parse_start = time.time()
            # OPTIMIZATION: Incremental parsing (O(1) per new token)
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
            parse_ms = (time.time() - parse_start) * 1000
            parse_wall_ms_list.append(parse_ms)

            batch_token_counts.append(len(new_token_ids))
            batch_wall_ms_value = (time.time() - t_iter) * 1000
            batch_wall_ms.append(batch_wall_ms_value)

            if emit_timing:
                logger.info(
                    "tts_stream_llm_batch",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "batch_index": iteration_count,
                        "batch_tokens": len(new_token_ids),
                        "llm_batch_wall_ms": int(batch_wall_ms_value),
                        "llm_batch_gpu_ms": int(batch_gpu_ms[-1]) if batch_gpu_ms else None,
                        "text_chunked": True,
                    },
                )

            # Capture global tokens once we have all 32
            if not captured_globals and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                captured_globals = global_buffer[:EXPECTED_GLOBAL_COUNT]

            # Process if we have all global tokens
            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                if t_first_chunk_ready is None and len(semantic_buffer) >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    t_first_chunk_ready = time.time()
                semantic_count = len(semantic_buffer)
                if semantic_count - last_decode_count >= DECODE_INTERVAL:
                    last_decode_count = semantic_count

                    all_semantic = semantic_buffer
                    window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]
                    decode_gpu_ms_this = None
                    t_decode_start = time.time()
                    if emit_timing and torch.cuda.is_available():
                        torch.cuda.synchronize()
                        decode_start_event = torch.cuda.Event(enable_timing=True)
                        decode_end_event = torch.cuda.Event(enable_timing=True)
                        decode_start_event.record()

                    if self.snac_decoder.enable_batching:
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=all_semantic,
                            global_ids=window_global,
                            trim_warmup=False,
                            use_sliding_window=False
                        )
                    else:
                        audio_bytes = self.snac_decoder.decode_streaming(
                            semantic_ids=all_semantic,
                            global_ids=window_global,
                            use_sliding_window=False,
                            trim_warmup=False
                        )

                    t_decode_end = time.time()
                    decode_calls += 1
                    decode_ms = (t_decode_end - t_decode_start) * 1000
                    decode_wall_ms += decode_ms
                    decode_wall_samples.append(decode_ms)
                    if emit_timing and torch.cuda.is_available():
                        decode_end_event.record()
                        torch.cuda.synchronize()
                        decode_gpu_ms_this = decode_start_event.elapsed_time(decode_end_event)
                        decode_gpu_ms += decode_gpu_ms_this
                        decode_gpu_samples.append(decode_gpu_ms_this)
                        batch_gpu_ms.append(decode_gpu_ms_this)

                    to_emit = b""
                    if audio_bytes:
                        total_samples_decoded = len(audio_bytes) // 2

                        if total_samples_decoded > total_samples_emitted_to_user:
                            new_bytes_start = total_samples_emitted_to_user * 2
                            new_audio_bytes = audio_bytes[new_bytes_start:]

                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                previous_chunk_tail,
                                new_audio_bytes,
                                sample_rate_hz=16000,
                                crossfade_ms=CROSSFADE_MS,
                            )

                            samples_emitted_in_this_chunk = len(to_emit) // 2
                            total_samples_emitted_to_user += samples_emitted_in_this_chunk

                    if to_emit:
                        total_audio_chunks += 1
                        if t_first_chunk_decoded is None:
                            t_first_chunk_decoded = t_decode_end
                        if emit_timing:
                            logger.info(
                                "tts_stream_decode_batch",
                                extra={
                                    "speaker": speaker,
                                    "request_id": request_id,
                                    "batch_index": decode_calls,
                                    "decode_wall_ms": int(decode_ms),
                                    "decode_gpu_ms": int(decode_gpu_ms_this)
                                    if decode_gpu_ms_this is not None else None,
                                    "semantic_count": len(semantic_buffer),
                                    "text_chunked": True,
                                },
                            )
                        yield (to_emit, captured_globals)

        # Final chunk processing
        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
            t_decode_start = time.time()
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,
                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
                use_sliding_window=False,
                trim_warmup=False
            )
            t_decode_end = time.time()
            decode_ms = (t_decode_end - t_decode_start) * 1000
            decode_calls += 1
            decode_wall_ms += decode_ms
            decode_wall_samples.append(decode_ms)

            to_emit = b""
            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,


Summary: auto


sed -n '860,1240p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 52cd04
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 4513
Output:
                        crossfade_ms=CROSSFADE_MS,
                    )

                    samples_emitted_in_final = len(to_emit) // 2
                    total_samples_emitted_to_user += samples_emitted_in_final

            if to_emit:
                total_audio_chunks += 1
                if t_first_chunk_decoded is None:
                    t_first_chunk_decoded = t_decode_end
                if emit_timing:
                    logger.info(
                        "tts_stream_decode_batch",
                        extra={
                            "speaker": speaker,
                            "request_id": request_id,
                            "batch_index": decode_calls,
                            "decode_wall_ms": int(decode_ms),
                            "decode_gpu_ms": None,
                            "semantic_count": len(semantic_buffer),
                            "text_chunked": True,
                        },
                    )
                yield (to_emit, captured_globals)

        # Flush remaining tail
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            yield (previous_chunk_tail, captured_globals)

        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000
        ttfb_ms = (
            (t_first_chunk_decoded - t_start) * 1000
            if t_first_chunk_decoded is not None
            else (t_first_chunk_ready - t_start) * 1000
            if t_first_chunk_ready is not None
            else (t_first_token - t_start) * 1000
            if t_first_token is not None
            else 0
        )
        request_metrics.update(
            {
                "request_id": request_id,
                "speaker": speaker,
                "generated_tokens": len(semantic_buffer) + len(global_buffer),
                "semantic_tokens": len(semantic_buffer),
                "global_tokens": len(captured_globals),
                "llm_batch_count": len(batch_wall_ms),
                "llm_batch_tokens": batch_token_counts,
                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_parse_ms": int(sum(parse_wall_ms_list)),
                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
                "decode_calls": decode_calls,
                "llm_decode_wall_ms_total": int(decode_wall_ms),
                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
                "llm_decode_gpu_ms_p50": int(statistics.median(decode_gpu_samples)) if decode_gpu_samples else 0,
                "audio_duration_seconds": audio_duration_s,
                "audio_chunks": total_audio_chunks,
                "captured_global_count": len(captured_globals),
                "text_chunked": True,
                "ttfb_ms": int(ttfb_ms),
                "generation_ms": int((t_end - t_start) * 1000),
            }
        )
        if batch_token_counts:
            request_metrics["tokens_per_batch_min"] = min(batch_token_counts)
            request_metrics["tokens_per_batch_max"] = max(batch_token_counts)
            request_metrics["tokens_per_batch_p50"] = int(statistics.median(batch_token_counts))

        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)

    async def generate_speech_stream_indic_continuation(
        self,
        speaker: str,
        text: str,
        global_ids: List[int],
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        request_id: Optional[str] = None,
        request_metrics: Optional[Dict[str, Any]] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech for CONTINUATION chunks using pre-captured global tokens.

        This is the key method for voice consistency in chunked generation:
        - Uses global_ids from the first chunk to maintain identical voice
        - Model generates only semantic tokens (skips global token generation)
        - Ensures no voice drift across chunks

        CRITICAL for production:
        - global_ids is request-scoped, passed explicitly (no shared state)
        - Thread-safe: each request has its own state
        - No global caching that could cause cross-request contamination

        Args:
            speaker: Speaker name (must match first chunk)
            text: Continuation text chunk to synthesize
            global_ids: 32 global token IDs captured from first chunk
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max tokens to generate
            repetition_penalty: Prevent repetition
            seed: Random seed (use same as first chunk for consistency)

        Yields:
            Audio bytes (int16 PCM, 16kHz mono)

        Raises:
            ValueError: If global_ids doesn't contain exactly 32 tokens

        Thread Safety:
            Fully thread-safe. All state is request-scoped and passed explicitly.
        """
        import time
        import torch

        t_start = time.time()
        emit_timing = _detailed_timing_enabled()
        request_metrics = request_metrics or {}
        request_metrics.update(
            {
                "request_id": request_id,
                "speaker": speaker,
                "text_length": len(text),
                "text_chunked": True,
            }
        )

        # Validate global tokens
        EXPECTED_GLOBAL_COUNT = 32
        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
            raise ValueError(
                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
                f"global_ids must be captured from first chunk generation."
            )

        # Build prompt WITH pre-filled global tokens
        # This tells the model to skip global generation and go straight to semantic tokens
        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)

        # Configure sampling
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,
            seed=seed,
        )

        # BiCodec token buffers
        semantic_buffer = []
        global_buffer_unused = []  # Not used in continuation, but needed for parser API
        processed_token_count = 0  # Track how many vLLM tokens we've parsed
        total_audio_chunks = 0

        # BiCodec streaming configuration
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
        CROSSFADE_MS = 50

        # Generate unique request ID
        import uuid
        if request_id is None:
            request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
            request_metrics["request_id"] = request_id

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser
        if token_parser is None:
            raise RuntimeError("BiCodecTokenParser unavailable for continuation streaming")

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track state
        batch_wall_ms: List[float] = []
        batch_gpu_ms: List[float] = []
        batch_token_counts: List[int] = []
        parse_wall_ms_list: List[float] = []
        decode_calls = 0
        decode_wall_ms = 0.0
        decode_gpu_ms = 0.0
        decode_wall_samples: List[float] = []
        decode_gpu_samples: List[float] = []
        t_first_token = None
        t_first_chunk_ready = None
        t_first_chunk_decoded = None
        iteration_count = 0
        last_decode_count = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None

        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
        async for request_output in results_generator:
            iteration_count += 1
            t_iter = time.time()
            if t_first_token is None:
                t_first_token = t_iter

            generated_ids = request_output.outputs[0].token_ids

            # OPTIMIZATION: Incremental parsing (O(1) per new token)
            # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
            parse_start = time.time()
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)
            parse_ms = (time.time() - parse_start) * 1000
            parse_wall_ms_list.append(parse_ms)
            batch_token_counts.append(len(new_token_ids))
            batch_wall_ms_value = (time.time() - t_iter) * 1000
            batch_wall_ms.append(batch_wall_ms_value)
            if emit_timing:
                logger.info(
                    "tts_stream_llm_batch",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "batch_index": iteration_count,
                        "batch_tokens": len(new_token_ids),
                        "llm_batch_wall_ms": int(batch_wall_ms_value),
                        "llm_batch_gpu_ms": int(batch_gpu_ms[-1]) if batch_gpu_ms else None,
                        "text_chunked": True,
                    },
                )

            semantic_count = len(semantic_buffer)
            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                if t_first_chunk_ready is None:
                    t_first_chunk_ready = time.time()
                if semantic_count - last_decode_count >= DECODE_INTERVAL:
                    last_decode_count = semantic_count

                    decode_gpu_ms_this = None
                    t_decode_start = time.time()
                    if emit_timing and torch.cuda.is_available():
                        torch.cuda.synchronize()
                        decode_start_event = torch.cuda.Event(enable_timing=True)
                        decode_end_event = torch.cuda.Event(enable_timing=True)
                        decode_start_event.record()

                    # Use pre-captured global tokens for decoding
                    if self.snac_decoder.enable_batching:
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=semantic_buffer,
                            global_ids=global_ids,  # Use captured globals
                            trim_warmup=False,
                            use_sliding_window=False
                        )
                    else:
                        audio_bytes = self.snac_decoder.decode_streaming(
                            semantic_ids=semantic_buffer,
                            global_ids=global_ids,  # Use captured globals
                            use_sliding_window=False,
                            trim_warmup=False
                        )

                    t_decode_end = time.time()
                    decode_calls += 1
                    decode_ms = (t_decode_end - t_decode_start) * 1000
                    decode_wall_ms += decode_ms
                    decode_wall_samples.append(decode_ms)
                    if emit_timing and torch.cuda.is_available():
                        decode_end_event.record()
                        torch.cuda.synchronize()
                        decode_gpu_ms_this = decode_start_event.elapsed_time(decode_end_event)
                        decode_gpu_ms += decode_gpu_ms_this
                        decode_gpu_samples.append(decode_gpu_ms_this)
                        batch_gpu_ms.append(decode_gpu_ms_this)

                    to_emit = b""
                    if audio_bytes:
                        total_samples_decoded = len(audio_bytes) // 2

                        if total_samples_decoded > total_samples_emitted_to_user:
                            new_bytes_start = total_samples_emitted_to_user * 2
                            new_audio_bytes = audio_bytes[new_bytes_start:]

                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                previous_chunk_tail,
                                new_audio_bytes,
                                sample_rate_hz=16000,
                                crossfade_ms=CROSSFADE_MS,
                            )

                            samples_emitted_in_this_chunk = len(to_emit) // 2
                            total_samples_emitted_to_user += samples_emitted_in_this_chunk

                    if to_emit:
                        total_audio_chunks += 1
                        if t_first_chunk_decoded is None:
                            t_first_chunk_decoded = t_decode_end
                        if emit_timing:
                            logger.info(
                                "tts_stream_decode_batch",
                                extra={
                                    "speaker": speaker,
                                    "request_id": request_id,
                                    "batch_index": decode_calls,
                                    "decode_wall_ms": int(decode_ms),
                                    "decode_gpu_ms": int(decode_gpu_ms_this)
                                    if decode_gpu_ms_this is not None else None,
                                    "semantic_count": len(semantic_buffer),
                                    "text_chunked": True,
                                },
                            )
                        yield to_emit

        # Final chunk processing
        if len(semantic_buffer) > last_decode_count:
            t_decode_start = time.time()
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,
                global_ids=global_ids,  # Use captured globals
                use_sliding_window=False,
                trim_warmup=False
            )
            t_decode_end = time.time()
            decode_ms = (t_decode_end - t_decode_start) * 1000
            decode_calls += 1
            decode_wall_ms += decode_ms
            decode_wall_samples.append(decode_ms)

            to_emit = b""
            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    samples_emitted_in_final = len(to_emit) // 2
                    total_samples_emitted_to_user += samples_emitted_in_final

                    if to_emit:
                        total_audio_chunks += 1
                        if t_first_chunk_decoded is None:
                            t_first_chunk_decoded = t_decode_end
                        if emit_timing:
                            logger.info(
                                "tts_stream_decode_batch",
                                extra={
                                    "speaker": speaker,
                                    "request_id": request_id,
                                    "batch_index": decode_calls,
                                    "decode_wall_ms": int(decode_ms),


sed -n '1240,1600p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 6547df
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 941
Output:
                                    "decode_wall_ms": int(decode_ms),
                                    "decode_gpu_ms": None,
                                    "semantic_count": len(semantic_buffer),
                                    "text_chunked": True,
                                },
                            )
                        yield to_emit

        # Flush remaining tail
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            yield previous_chunk_tail

        t_end = time.time()
        ttfb_ms = (
            (t_first_chunk_decoded - t_start) * 1000
            if t_first_chunk_decoded is not None
            else (t_first_chunk_ready - t_start) * 1000
            if t_first_chunk_ready is not None
            else (t_first_token - t_start) * 1000
            if t_first_token is not None
            else 0
        )
        request_metrics.update(
            {
                "request_id": request_id,
                "speaker": speaker,
                "generated_tokens": len(semantic_buffer) + len(global_ids),
                "semantic_tokens": len(semantic_buffer),
                "global_tokens": len(global_ids),
                "llm_batch_count": len(batch_wall_ms),
                "llm_batch_tokens": batch_token_counts,
                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
                "llm_parse_ms": int(sum(parse_wall_ms_list)),
                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
                "decode_calls": decode_calls,
                "llm_decode_wall_ms_total": int(decode_wall_ms),
                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
                "llm_decode_gpu_ms_p50": int(statistics.median(decode_gpu_samples)) if decode_gpu_samples else 0,
                "audio_duration_seconds": total_samples_emitted_to_user / 16000,
                "audio_chunks": total_audio_chunks,
                "text_chunked": True,
                "ttfb_ms": int(ttfb_ms),
                "generation_ms": int((t_end - t_start) * 1000),
            }
        )
        if batch_token_counts:
            request_metrics["tokens_per_batch_min"] = min(batch_token_counts)
            request_metrics["tokens_per_batch_max"] = max(batch_token_counts)
            request_metrics["tokens_per_batch_p50"] = int(statistics.median(batch_token_counts))
        logger.info("tts_stream_continuation_complete", extra=request_metrics)


rg -n "import vllm|AsyncLLMEngine|VEENA3_TIMING_LOGS|VEENA3_FORCE_CPU|vllm" veena3modal | head -n 300


Output:
Chunk ID: 84583f
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 242
Output:
veena3modal/app.py:43:        "vllm>=0.4.0",
veena3modal/services/tts_runtime.py:28:_PROFILE_REQUEST_TIMING = "VEENA3_TIMING_LOGS"
veena3modal/services/tts_runtime.py:289:    from vllm import SamplingParams
veena3modal/core/streaming_pipeline.py:30:from vllm import SamplingParams
veena3modal/core/streaming_pipeline.py:54:    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
veena3modal/core/pipeline.py:15:from vllm import SamplingParams
veena3modal/core/pipeline.py:36:      VEENA3_TIMING_LOGS=true|1
veena3modal/core/pipeline.py:38:    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
veena3modal/core/model_loader.py:12:from vllm import AsyncLLMEngine, AsyncEngineArgs, SamplingParams
veena3modal/core/model_loader.py:128:        self.engine = AsyncLLMEngine.from_engine_args(engine_args)
veena3modal/core/multi_engine.py:29:    Wraps multiple AsyncLLMEngine instances with round-robin dispatch.


sed -n '1,220p' veena3modal/core/model_loader.py


Output:
Chunk ID: c626d9
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1800
Output:
"""
Spark TTS Model Loader

Loads Spark TTS model with vLLM engine and validates speakers/emotions.
Migrated from Veena3/Orpheus to Spark TTS architecture.
"""

import os
import torch
from typing import Optional, Dict, Any
from transformers import AutoTokenizer
from vllm import AsyncLLMEngine, AsyncEngineArgs, SamplingParams

from veena3modal.core.constants import (
    INDIC_EMOTION_TAGS,
    INDIC_SPEAKERS,
    SPEAKER_MAP,
    DEFAULT_MODEL_PATH,
    DEFAULT_MAX_MODEL_LEN,
    VLLM_CONFIG,
)

class SparkTTSModel:
    """
    Spark TTS Model with vLLM inference engine and BiCodec audio tokenizer.

    Model: BayAreaBoys/spark_tts_4speaker
    Architecture: Qwen2ForCausalLM with BiCodec audio tokenization
    Speakers: 12 predefined (mapped to speaker_0-11 as per training)
    Emotions: 10 tags in [bracket] format

    CRITICAL REQUIREMENTS:
    - Load tokenizer with special Spark TTS tokens
    - Configure vLLM with optimizations (prefix caching, CUDA graphs)
    - Support speaker mapping (lipakshi → speaker_0, etc.)
    """

    def __init__(
        self,
        model_path: str = None,
        model_type: str = "indic_speakers",  # Default to indic_speakers for backward compatibility
        hf_token: str = None,
        **engine_kwargs
    ):
        """
        Initialize Spark TTS model with vLLM.

        Args:
            model_path: Path to model directory (local or HF). If None, uses env var or default.
            model_type: Model type (kept for compatibility, always uses indic_speakers)
            hf_token: HuggingFace token for private models
            **engine_kwargs: Additional vLLM engine arguments (override defaults from VLLM_CONFIG)
        """
        self.model_type = "indic_speakers"  # Spark TTS only supports speaker-based

        # Use provided path, or check environment variable, or use default
        if model_path is None:
            model_path = os.environ.get(
                'SPARK_TTS_MODEL_PATH',
                DEFAULT_MODEL_PATH
            )

        self.model_path = model_path
        self.hf_token = hf_token

        print(f"🚀 Initializing Spark TTS Model")
        print(f"📁 Model path: {model_path}")
        print("🗣  Speakers: 12 predefined "
              "(lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, "
              "Nilay, Aarvi, Asha, Bittu, Mira)")
        print(f"🎭 Emotions: 10 tags in [bracket] format")

        # Load tokenizer
        print(f"\n📝 Loading tokenizer from model...")
        tokenizer_kwargs = {
            'trust_remote_code': True,
        }

        # Add HF token for private repos
        if hf_token:
            tokenizer_kwargs['token'] = hf_token

        # Add emotion tags as special tokens
        special_tokens = {
            'additional_special_tokens': INDIC_EMOTION_TAGS
        }

        self.tokenizer = AutoTokenizer.from_pretrained(
            model_path,
            **tokenizer_kwargs
        )

        # Add special tokens
        self.tokenizer.add_special_tokens(special_tokens)

        print(f"✅ Tokenizer loaded: {len(self.tokenizer)} tokens")

        # Validate speakers and emotions
        self._validate_speakers()
        self._validate_emotion_tags()

        # Initialize vLLM engine with Spark TTS optimizations
        print(f"\n🔧 Initializing vLLM engine with optimizations...")

        # Start with default VLLM_CONFIG and override with kwargs
        engine_kwargs_dict = VLLM_CONFIG.copy()
        engine_kwargs_dict.update({
            'model': model_path,
            'tokenizer': model_path,
        })

        # Add HF token for private models
        if hf_token:
            os.environ['HUGGING_FACE_HUB_TOKEN'] = hf_token

        # Override with any user-provided kwargs
        engine_kwargs_dict.update(engine_kwargs)

        # Print configuration
        print(f"  - dtype: {engine_kwargs_dict['dtype']}")
        print(f"  - max_model_len: {engine_kwargs_dict['max_model_len']}")
        print(f"  - gpu_memory_utilization: {engine_kwargs_dict['gpu_memory_utilization']}")
        print(f"  - enable_prefix_caching: {engine_kwargs_dict.get('enable_prefix_caching', False)}")
        print(f"  - enforce_eager (disable CUDA graphs): {engine_kwargs_dict.get('enforce_eager', True)}")

        engine_args = AsyncEngineArgs(**engine_kwargs_dict)
        self.engine = AsyncLLMEngine.from_engine_args(engine_args)

        print(f"✅ vLLM engine initialized successfully!")
        print(f"\n🎉 Spark TTS Model ready for inference!\n")

    def _validate_emotion_tags(self):
        """
        Validate that emotion tags are properly configured for Spark TTS.

        For Spark TTS, emotion tags use [bracket] format and should be
        added as special tokens to the tokenizer.
        """
        print(f"\n🔍 Validating {len(INDIC_EMOTION_TAGS)} emotion tags for Spark TTS...")

        # Check if emotion tags can be tokenized
        for tag in INDIC_EMOTION_TAGS:
            token_ids = self.tokenizer.encode(tag, add_special_tokens=False)
            # Spark TTS emotion tags may be multiple tokens, that's okay
            if not token_ids:
                print(f"⚠  Warning: Emotion tag {tag} produced no tokens")

        print(f"✅ All {len(INDIC_EMOTION_TAGS)} emotion tags validated")

        # Print sample emotion tags
        print(f"\n📋 Sample emotion tags:")
        for i, tag in enumerate(INDIC_EMOTION_TAGS[:5]):
            token_ids = self.tokenizer.encode(tag, add_special_tokens=False)
            print(f"  {tag}: {len(token_ids)} tokens")
        if len(INDIC_EMOTION_TAGS) > 5:
            print(f"  ...")

    def _validate_speakers(self):
        """
        Validate that all speakers are properly mapped for Spark TTS.

        Spark TTS uses speaker tokens like <|speaker_0|> to <|speaker_7|>.
        User-facing names (lipakshi, vardan, etc.) are mapped to these IDs.
        """
        print(f"\n🔍 Validating {len(INDIC_SPEAKERS)} speakers for Spark TTS...")

        # Validate speaker mapping
        for speaker_name in INDIC_SPEAKERS:
            if speaker_name not in SPEAKER_MAP:
                print(f"⚠  Warning: Speaker {speaker_name} not in SPEAKER_MAP")
                continue

            speaker_id = SPEAKER_MAP[speaker_name]
            speaker_token = f'<|speaker_{speaker_id}|>'
            token_ids = self.tokenizer.encode(speaker_token, add_special_tokens=False)
            print(f"  {speaker_name} → {speaker_token}: {len(token_ids)} tokens")

        print(f"✅ All {len(INDIC_SPEAKERS)} speakers validated with mapping")

    def get_model_type(self) -> str:
        """Get the model type (always 'indic_speakers' for Spark TTS)."""
        return self.model_type

    def get_tokenizer(self):
        """Get the tokenizer."""
        return self.tokenizer

    def get_engine(self):
        """Get the vLLM engine."""
        return self.engine


I’m checking whether we already have previous benchmark artifacts in-repo, then I’ll run the stress script end-to-end if the local server can initialize; if blocked by the env, I’ll give the exact failure and the minimal path to unblock it.


rg -n "stress_test_optimized_local|speed|throughput|concurrency|Benchmark|stress" -g "*.json" -g "*.md" -g "*.log"


Output:
Chunk ID: 2cf418
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 1540
Output:
modal_docs/Reference/# modal.Image.md:21:which speeds up deployment.
modal_docs/Reference/# modal.Image.md:50:which speeds up deployment.
modal_docs/Reference/# modal.Image.md:114:which speeds up deployment.
modal_docs/Reference/# modal.file_io.md:14:may be limited in size to 100 MiB, and the throughput of requests is
modal_docs/Reference/# modal.concurrent.md:6:    max_inputs: Optional[int] = None,  # Hard limit on each container's input concurrency
modal_docs/Reference/# modal.concurrent.md:7:    target_inputs: Optional[int] = None,  # Input concurrency that Modal's autoscaler should target
modal_docs/Reference/# modal.concurrent.md:16:The concurrency mechanism depends on whether the function is async or not:
modal_docs/Reference/# modal.concurrent.md:20:Input concurrency will be most useful for workflows that are IO-bound
modal_docs/Reference/# modal.concurrent.md:27:if resources are insufficient to remain at the target concurrency, e.g. when the
modal_docs/Reference/# modal.concurrent.md:33:# Stack the decorator under `@app.function()` to enable input concurrency
modal_docs/Reference/# modal.Cls.md:70:    concurrency_limit: Optional[int] = None,  # Now called `max_containers`
modal_docs/Reference/# modal.Cls.md:72:    allow_concurrent_inputs: Optional[int] = None,  # See `.with_concurrency`
modal_docs/Reference/# modal.Cls.md:102:## with_concurrency
modal_docs/Reference/# modal.Cls.md:105:def with_concurrency(self: "_Cls", *, max_inputs: int, target_inputs: Optional[int] = None) -> "_Cls":
modal_docs/Reference/# modal.Cls.md:108:Create an instance of the Cls with input concurrency enabled or overridden with new values.
modal_docs/Reference/# modal.Cls.md:114:ModelUsingGPU = Model.with_options(gpu="A100").with_concurrency(max_inputs=100)
modal_docs/Reference/# modal.Cls.md:115:ModelUsingGPU().generate.remote(42)  # will run on an A100 GPU with input concurrency enabled
modal_docs/Reference/# modal.Cls.md:130:ModelUsingGPU().generate.remote(42)  # will run on an A100 GPU with input concurrency enabled
modal_docs/# Scaling out.md:205:Sometimes you can speed up your applications by utilizing GPU acceleration. See
modal_docs/Reference/# modal App.md:322:    concurrency_limit: Optional[int] = None,  # Replaced with `max_containers`
modal_docs/Reference/# modal App.md:386:    concurrency_limit: Optional[int] = None,  # Replaced with `max_containers`
modal_docs/# Cold start performance.md:76:It is not always possible to speed up boots sufficiently.
modal_docs/# Cold start performance.md:203:Sometimes, there is nothing to be done but to speed this work up.
modal_docs/# Cold start performance.md:256:If performing concurrent IO on large file reads does _not_ speed up your cold
modal_docs/# QuiLLMan: Voice Chat with Moshi.md:38:Traditionally, a speech-to-speech chat app requires three distinct modules: speech-to-text, text-to-text, and text-to-speech. Passing data between these modules introduces bottlenecks, and can limit the speed of the app and forces a turn-by-turn conversation which can feel unnatural.
modal_docs/# Input concurrency.md:1:# Input concurrency
modal_docs/# Input concurrency.md:15:one-by-one. Modal supports these workloads with its _input concurrency_ feature,
modal_docs/# Input concurrency.md:20:When used effectively, input concurrency can reduce latency and lower costs.
modal_docs/# Input concurrency.md:24:Input concurrency can be especially effective for workloads that are primarily
modal_docs/# Input concurrency.md:42:Note that for CPU-bound workloads, input concurrency will likely not be as
modal_docs/# Input concurrency.md:46:## Enabling input concurrency
modal_docs/# Input concurrency.md:48:To enable input concurrency, add the `@modal.concurrent` decorator:
modal_docs/# Input concurrency.md:72:with input concurrency enabled will concurrently run distinct methods in
modal_docs/# Input concurrency.md:76:Python SDK. Input concurrency could previously be enabled by setting the
modal_docs/# Input concurrency.md:79:## Setting a concurrency target
modal_docs/# Input concurrency.md:107:Modal uses different concurrency mechanisms to execute your Function depending
modal_docs/# Input concurrency.md:109:certain requirements on the Function implementation. Input concurrency is an
modal_docs/# Input concurrency.md:141:Input concurrency is a powerful feature, but there are a few caveats that can
modal_docs/# Input concurrency.md:150:When using input concurrency with a synchronous Function, a single input
modal_docs/# Dynamic batching (beta).md:6:Batching increases throughput at a potential cost to latency.
modal_docs/# Dynamic batching (beta).md:9:as GPUs are designed to maximize throughput and are frequently bottlenecked on shareable resources,
modal_docs/# Dynamic batching (beta).md:14:maximizing the throughput benefit of batching while minimizing the latency penalty.
modal_docs/# Run a FastRTC app on Modal.md:153:                concurrency_limit=MAX_CONCURRENT_STREAMS,  # limit simultaneous connections
modal_docs/# Proxies (beta).md:66:throughput linearly.
modal_docs/# GPU acceleration.md:92:Since arithmetic throughput has risen faster than memory throughput in recent
modal_docs/# GPU acceleration.md:93:hardware generations, speedups for memory-bound GPU jobs are not as extreme and
modal_docs/# Volumes.md:60:Actual throughput is not guaranteed and may be lower depending on network conditions.
modal_docs/# Volumes.md:396:Volumes v2 support more files, higher throughput, and more irregular access
modal_docs/# Fast pull from registry.md:43:Using estargz to speed up image pull (index loaded in 1.86s)...
modal_docs/# Real-time object detection with WebRTC.md:63:To ensure we properly leverage Modal's autoscaling and concurrency features, we need to align the signaling and streaming lifetimes with Modal Function Call lifetimes.
modal_docs/# Real-time object detection with WebRTC.md:79:That way, our WebRTC application benefits from all the autoscaling and concurrency logic built into Modal
modal_docs/# Real-time object detection with WebRTC.md:190:the Internet operates within a factor of two of the speed of light, but that's just not that fast.


Output:
Chunk ID: c703dd
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 20
Output:
README.md
external
modal_docs
models
requirements.txt
scripts
veena3modal
venv


I’ve confirmed the timing surface is present but still incomplete for some non-streaming decode fields. Next I’m going to attempt a local benchmark run to capture real concurrency measurements; if vLLM import issues persist, I’ll give the exact fix path first.


python3 -m veena3modal.local_server --skip-download --port 8000


Output:
Chunk ID: 45ddc1
Wall time: 1.2011 seconds
Process running with session ID 26117
Original token count: 454
Output:
2026-02-17 00:10:52 | INFO     | veena3.local | Auth bypass enabled (local mode). Use --auth to require API keys.
2026-02-17 00:10:52 | INFO     | veena3.local | Resolved model paths:
2026-02-17 00:10:52 | INFO     | veena3.local |   LLM (vLLM):    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:10:52 | INFO     | veena3.local |   BiCodec:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:10:52 | INFO     | veena3.local | ============================================================
2026-02-17 00:10:52 | INFO     | veena3.local | Initializing Veena3 TTS Runtime (local)
2026-02-17 00:10:52 | INFO     | veena3.local |   Model:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:10:52 | INFO     | veena3.local |   LLM path:    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:10:52 | INFO     | veena3.local |   BiCodec:     /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:10:52 | INFO     | veena3.local |   Device:      cuda
2026-02-17 00:10:52 | INFO     | veena3.local |   GPU Memory:  25% (total)
2026-02-17 00:10:52 | INFO     | veena3.local |   Engines:     1
2026-02-17 00:10:52 | INFO     | veena3.local |   SR:          disabled
2026-02-17 00:10:52 | INFO     | veena3.local | ============================================================
2026-02-17 00:10:52 | INFO     | veena3modal.services.tts_runtime | Initializing TTS runtime...
2026-02-17 00:10:52 | INFO     | veena3modal.services.tts_runtime |   Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:10:52 | INFO     | veena3modal.services.tts_runtime |   BiCodec path: /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:10:52 | INFO     | veena3modal.services.tts_runtime |   SR enabled: False, path: None


Output:
Chunk ID: a185cc
Wall time: 1.5129 seconds
Process exited with code 1
Original token count: 1586
Output:
2026-02-17 00:10:56 | ERROR    | veena3modal.services.tts_runtime | ❌ Failed to initialize TTS runtime: /home/ubuntu/.local/lib/python3.10/site-packages/vllm/_C.abi3.so: undefined symbol: _ZN3c104cuda29c10_cuda_check_implementationEiPKcS2_ib
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 146, in initialize_runtime
    from veena3modal.core.model_loader import SparkTTSModel
  File "/home/ubuntu/veenaModal/veena3modal/core/model_loader.py", line 12, in <module>
    from vllm import AsyncLLMEngine, AsyncEngineArgs, SamplingParams
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/__init__.py", line 74, in __getattr__
    module = import_module(module_name, __package__)
  File "/usr/lib/python3.10/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 4, in <module>
    from vllm.v1.engine.async_llm import AsyncLLM
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/__init__.py", line 14, in <module>
    from vllm.multimodal.inputs import MultiModalFeatureSpec
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/__init__.py", line 14, in <module>
    from .registry import MultiModalRegistry
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/registry.py", line 8, in <module>
    from vllm.config.multimodal import BaseDummyOptions
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/config/__init__.py", line 5, in <module>
    from vllm.config.cache import CacheConfig
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/config/cache.py", line 14, in <module>
    from vllm.utils.mem_utils import format_gib, get_cpu_memory
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/utils/mem_utils.py", line 14, in <module>
    from vllm.platforms import current_platform
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/platforms/__init__.py", line 257, in __getattr__
    _current_platform = resolve_obj_by_qualname(platform_cls_qualname)()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/utils/import_utils.py", line 111, in resolve_obj_by_qualname
    module = importlib.import_module(module_name)
  File "/usr/lib/python3.10/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/platforms/cuda.py", line 16, in <module>
    import vllm._C  # noqa
ImportError: /home/ubuntu/.local/lib/python3.10/site-packages/vllm/_C.abi3.so: undefined symbol: _ZN3c104cuda29c10_cuda_check_implementationEiPKcS2_ib
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 146, in initialize_runtime
    from veena3modal.core.model_loader import SparkTTSModel
  File "/home/ubuntu/veenaModal/veena3modal/core/model_loader.py", line 12, in <module>
    from vllm import AsyncLLMEngine, AsyncEngineArgs, SamplingParams
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/__init__.py", line 74, in __getattr__
    module = import_module(module_name, __package__)
  File "/usr/lib/python3.10/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/engine/async_llm_engine.py", line 4, in <module>
    from vllm.v1.engine.async_llm import AsyncLLM
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/__init__.py", line 14, in <module>
    from vllm.multimodal.inputs import MultiModalFeatureSpec
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/__init__.py", line 14, in <module>
    from .registry import MultiModalRegistry
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/registry.py", line 8, in <module>
    from vllm.config.multimodal import BaseDummyOptions
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/config/__init__.py", line 5, in <module>
    from vllm.config.cache import CacheConfig
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/config/cache.py", line 14, in <module>
    from vllm.utils.mem_utils import format_gib, get_cpu_memory
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/utils/mem_utils.py", line 14, in <module>
    from vllm.platforms import current_platform
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/platforms/__init__.py", line 257, in __getattr__
    _current_platform = resolve_obj_by_qualname(platform_cls_qualname)()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/utils/import_utils.py", line 111, in resolve_obj_by_qualname
    module = importlib.import_module(module_name)
  File "/usr/lib/python3.10/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/platforms/cuda.py", line 16, in <module>
    import vllm._C  # noqa
ImportError: /home/ubuntu/.local/lib/python3.10/site-packages/vllm/_C.abi3.so: undefined symbol: _ZN3c104cuda29c10_cuda_check_implementationEiPKcS2_ib

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/home/ubuntu/veenaModal/veena3modal/local_server.py", line 465, in <module>
    main()
  File "/home/ubuntu/veenaModal/veena3modal/local_server.py", line 426, in main
    initialize_local_runtime(
  File "/home/ubuntu/veenaModal/veena3modal/local_server.py", line 224, in initialize_local_runtime
    runtime = initialize_runtime(
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 272, in initialize_runtime
    raise RuntimeError(f"TTS runtime initialization failed: {e}") from e
RuntimeError: TTS runtime initialization failed: /home/ubuntu/.local/lib/python3.10/site-packages/vllm/_C.abi3.so: undefined symbol: _ZN3c104cuda29c10_cuda_check_implementationEiPKcS2_ib


python3 - <<'PY'
import torch,sys
import pkg_resources
print('torch',torch.__version__)
try:
    import vllm
    print('vllm',vllm.__version__)
except Exception as e:
    print('vllm_import_error',repr(e))
PY


Output:
Chunk ID: a5924c
Wall time: 2.9279 seconds
Process exited with code 0
Original token count: 8
Output:
torch 2.10.0+cu128
vllm 0.15.1


Output:
Chunk ID: 3fe2d7
Wall time: 0.6149 seconds
Process exited with code 0
Original token count: 253
Output:
Name: vllm
Version: 0.15.1
Summary: A high-throughput and memory-efficient inference and serving engine for LLMs
Home-page: https://github.com/vllm-project/vllm
Author: vLLM Team
Author-email:
License-Expression: Apache-2.0
Location: /home/ubuntu/.local/lib/python3.10/site-packages
Requires: aiohttp, anthropic, blake3, cachetools, cbor2, cloudpickle, compressed-tensors, depyf, diskcache, einops, fastapi, filelock, flashinfer-python, gguf, grpcio, grpcio-reflection, ijson, lark, llguidance, lm-format-enforcer, mcp, mistral_common, model-hosting-container-standards, msgspec, ninja, numba, numpy, openai, openai-harmony, opencv-python-headless, outlines_core, partial-json-parser, pillow, prometheus-fastapi-instrumentator, prometheus_client, protobuf, psutil, py-cpuinfo, pybase64, pydantic, python-json-logger, pyyaml, pyzmq, ray, regex, requests, sentencepiece, setproctitle, tiktoken, tokenizers, torch, torchaudio, torchvision, tqdm, transformers, typing_extensions, watchfiles, xgrammar
Required-by:


python3 -m pip index versions vllm | sed -n '1,120p'


Output:
Chunk ID: 5cd6bc
Wall time: 0.7150 seconds
Process exited with code 0
Original token count: 167
Output:
vllm (0.15.1)
Available versions: 0.15.1, 0.15.0, 0.14.1, 0.14.0, 0.13.0, 0.12.0, 0.11.2, 0.11.1, 0.11.0, 0.10.2, 0.10.1.1, 0.10.1, 0.10.0, 0.9.2, 0.9.1, 0.9.0.1, 0.9.0, 0.8.5.post1, 0.8.5, 0.8.4, 0.8.3, 0.8.2, 0.8.1, 0.8.0, 0.7.3, 0.7.2, 0.7.1, 0.7.0, 0.6.6.post1, 0.6.6, 0.6.5, 0.6.4.post1, 0.6.4, 0.6.3.post1, 0.6.3, 0.6.2, 0.6.1.post2, 0.6.1.post1, 0.6.1, 0.6.0, 0.5.5, 0.5.4, 0.5.3.post1, 0.5.3, 0.5.2, 0.5.1, 0.5.0.post1, 0.5.0, 0.4.3, 0.4.2, 0.4.1, 0.4.0.post1, 0.4.0, 0.3.3, 0.3.2, 0.3.1, 0.3.0, 0.2.7, 0.2.6, 0.2.5, 0.2.4, 0.2.3, 0.2.2, 0.2.1.post1, 0.2.0, 0.1.7, 0.1.6, 0.1.5, 0.1.4, 0.1.3, 0.1.2, 0.1.1, 0.1.0, 0.0.1
  INSTALLED: 0.15.1
  LATEST:    0.15.1


nvidia-smi --query-gpu=driver_version,name | sed -n '1,5p'


Output:
Chunk ID: 59ff8b
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 18
Output:
"--format=" switch is missing. Please run 'nvidia-smi -h' for help.


Output:
Chunk ID: 86305f
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 9
Output:
560.35.05, NVIDIA A100-SXM4-80GB


python3 - <<'PY'
import importlib.metadata as md
md = md.metadata('vllm')
print(md.get_all('Requires-Dist'))
PY


Output:
Chunk ID: 1bed8f
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 510
Output:
['regex', 'cachetools', 'psutil', 'sentencepiece', 'numpy', 'requests>=2.26.0', 'tqdm', 'blake3', 'py-cpuinfo', 'transformers<5,>=4.56.0', 'tokenizers>=0.21.1', 'protobuf>=6.33.5', 'fastapi[standard]>=0.115.0', 'aiohttp>=3.13.3', 'openai>=1.99.1', 'pydantic>=2.12.0', 'prometheus_client>=0.18.0', 'pillow', 'prometheus-fastapi-instrumentator>=7.0.0', 'tiktoken>=0.6.0', 'lm-format-enforcer==0.11.3', 'llguidance<1.4.0,>=1.3.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "s390x" or platform_machine == "ppc64le"', 'outlines_core==0.2.11', 'diskcache==5.6.3', 'lark==1.2.2', 'xgrammar==0.1.29; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"', 'typing_extensions>=4.10', 'filelock>=3.16.1', 'partial-json-parser', 'pyzmq>=25.0.0', 'msgspec', 'gguf>=0.17.0', 'mistral_common[image]>=1.8.8', 'opencv-python-headless>=4.13.0', 'pyyaml', 'six>=1.16.0; python_version > "3.11"', 'setuptools<81.0.0,>=77.0.3; python_version > "3.11"', 'einops', 'compressed-tensors==0.13.0', 'depyf==0.20.0', 'cloudpickle', 'watchfiles', 'python-json-logger', 'ninja', 'pybase64', 'cbor2', 'ijson', 'setproctitle', 'openai-harmony>=0.0.3', 'anthropic>=0.71.0', 'model-hosting-container-standards<1.0.0,>=0.1.13', 'mcp', 'grpcio', 'grpcio-reflection', 'numba==0.61.2', 'ray[cgraph]>=2.48.0', 'torch==2.9.1', 'torchaudio==2.9.1', 'torchvision==0.24.1', 'flashinfer-python==0.6.1', 'pandas; extra == "bench"', 'matplotlib; extra == "bench"', 'seaborn; extra == "bench"', 'datasets; extra == "bench"', 'scipy; extra == "bench"', 'tensorizer==2.10.1; extra == "tensorizer"', 'fastsafetensors>=0.1.10; extra == "fastsafetensors"', 'runai-model-streamer[gcs,s3]>=0.15.3; extra == "runai"', 'librosa; extra == "audio"', 'scipy; extra == "audio"', 'soundfile; extra == "audio"', 'mistral_common[audio]; extra == "audio"', 'petit-kernel; extra == "petit-kernel"', 'helion; extra == "helion"']


python3 -m pip install --dry-run torch==2.9.1 -q


Output:
Chunk ID: 5fc26c
Wall time: 1.0717 seconds
Process exited with code 0
Original token count: 0
Output:


python3 -m pip install --dry-run torch==2.9.1


Output:
Chunk ID: a714be
Wall time: 0.9491 seconds
Process exited with code 0
Original token count: 929
Output:
Defaulting to user installation because normal site-packages is not writeable
Looking in indexes: https://mcache-dsm.massedcompute.com/simple, https://pypi.org/simple
Collecting torch==2.9.1
  Using cached torch-2.9.1-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (30 kB)
Requirement already satisfied: filelock in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (3.24.2)
Requirement already satisfied: typing-extensions>=4.10.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (4.15.0)
Requirement already satisfied: sympy>=1.13.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (1.14.0)
Requirement already satisfied: networkx>=2.5.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (3.4.2)
Requirement already satisfied: jinja2 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (3.1.6)
Requirement already satisfied: fsspec>=0.8.5 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (2026.2.0)
Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.8.93 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (12.8.93)
Requirement already satisfied: nvidia-cuda-runtime-cu12==12.8.90 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (12.8.90)
Requirement already satisfied: nvidia-cuda-cupti-cu12==12.8.90 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (12.8.90)
Requirement already satisfied: nvidia-cudnn-cu12==9.10.2.21 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (9.10.2.21)
Requirement already satisfied: nvidia-cublas-cu12==12.8.4.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (12.8.4.1)
Requirement already satisfied: nvidia-cufft-cu12==11.3.3.83 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (11.3.3.83)
Requirement already satisfied: nvidia-curand-cu12==10.3.9.90 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (10.3.9.90)
Requirement already satisfied: nvidia-cusolver-cu12==11.7.3.90 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (11.7.3.90)
Requirement already satisfied: nvidia-cusparse-cu12==12.5.8.93 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (12.5.8.93)
Requirement already satisfied: nvidia-cusparselt-cu12==0.7.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (0.7.1)
Requirement already satisfied: nvidia-nccl-cu12==2.27.5 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (2.27.5)
Collecting nvidia-nvshmem-cu12==3.3.20 (from torch==2.9.1)
  Using cached nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (2.1 kB)
Requirement already satisfied: nvidia-nvtx-cu12==12.8.90 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (12.8.90)
Requirement already satisfied: nvidia-nvjitlink-cu12==12.8.93 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (12.8.93)
Requirement already satisfied: nvidia-cufile-cu12==1.13.1.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from torch==2.9.1) (1.13.1.3)
Collecting triton==3.5.1 (from torch==2.9.1)
  Using cached triton-3.5.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl.metadata (1.7 kB)
Requirement already satisfied: mpmath<1.4,>=1.1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from sympy>=1.13.3->torch==2.9.1) (1.3.0)
Requirement already satisfied: MarkupSafe>=2.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from jinja2->torch==2.9.1) (3.0.3)
Would install nvidia-nvshmem-cu12-3.3.20 torch-2.9.1 triton-3.5.1


python3 - <<'PY'
import torch, importlib.metadata as md
for pkg in ['torch','torchvision','torchaudio','triton','nvidia-cublas-cu12','nvidia-cuda-runtime-cu12']:
    try:
        v=md.version(pkg)
    except Exception:
        v='n/a'
    print(pkg,v)
PY


Output:
Chunk ID: 0f11ad
Wall time: 1.6941 seconds
Process exited with code 0
Original token count: 30
Output:
torch 2.10.0
torchvision n/a
torchaudio n/a
triton 3.6.0
nvidia-cublas-cu12 12.8.4.1
nvidia-cuda-runtime-cu12 12.8.90


I reproduced the vLLM failure: symbol mismatch from `vllm _C` because installed `torch` is 2.10.0 and `vllm 0.15.1` hard-pins `torch==2.9.1`. I’ll quickly align to `torch 2.9.1` (and compatible CUDA wheels already on this host) and then rerun the local server + stress test.


python3 -m pip install torch==2.9.1 torchaudio==2.9.1 torchvision==0.24.1