Output:
Chunk ID: cba431
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 1131
Output:
        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
        needs_chunking = enable_chunking and long_processor.should_chunk(text)

        if needs_chunking:
            # Chunked streaming with voice consistency (global token caching)
            metrics["text_chunked"] = True
            async for audio_chunk, chunk_metrics in _stream_chunked_text(
                runtime,
                long_processor,
                text,
                speaker,
                temperature,
                top_k,
                top_p,
                max_tokens,
                repetition_penalty,
                seed,
                sample_rate,
                request_id,
                request_metrics=pipeline_metrics,
            ):
                if first_chunk_time is None:
                    first_chunk_time = time.time()
                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                    # Create streaming WAV header (size=0 for unknown length)
                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
                    audio_chunk = wav_header + audio_chunk

                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
                metrics["chunks_sent"] += 1
                metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
                if chunk_metrics:
                    metrics.update(chunk_metrics)

                yield audio_chunk, metrics
        else:
            # Simple streaming (no text chunking)
            # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
            # use continuation mode to skip the ~110ms global token pre-roll.
            # The model jumps straight to semantic token generation.
            cached_globals = runtime.speaker_global_cache.get(speaker)

            if cached_globals:
                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                    speaker=speaker,
                    text=text,
                    global_ids=cached_globals,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    request_id=request_id,
                    request_metrics=pipeline_metrics,
                )
            else:
                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
                    speaker=speaker,
                    text=text,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    request_id=request_id,
                    request_metrics=pipeline_metrics,
                )

            async for audio_chunk in stream_gen:
                if first_chunk_time is None:
                    first_chunk_time = time.time()
                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
                    audio_chunk = wav_header + audio_chunk

                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
                metrics["chunks_sent"] += 1
                metrics["total_bytes"] = total_pcm_bytes + 44
                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
                yield audio_chunk, metrics
    finally:
        active_requests = _with_inflight_delta(-1)
        metrics.update(pipeline_metrics)
        metrics["generation_ms"] = int((time.time() - start_time) * 1000)
        metrics["text_chunked"] = bool(metrics.get("text_chunked", False))
        if _timing_enabled():
            logger.info(
                "tts_runtime_stream_end",
                extra={
                    "speaker": speaker,
                    "request_id": request_id,
                    "request_inflight": active_requests,
                    "chunks_sent": metrics.get("chunks_sent", 0),
                    "generation_ms": metrics["generation_ms"],
                },
            )


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '1,180p' veena3modal/services/tts_runtime.py


Output:
Chunk ID: 4bcc53
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 1587
Output:
"""
TTS runtime wrapper (container-scoped singleton).

In Modal, each container should load the model once and then handle many requests.
This wrapper owns:
- SparkTTSModel (vLLM engine)
- IndicPromptBuilder
- BiCodecDecoder
- SparkTTSPipeline + Veena3SlidingWindowPipeline
- optional SuperResolutionService

Imports framework-agnostic inference + processing code from `veena3modal/core`,
`veena3modal/processing`, and `veena3modal/audio`.
"""

from __future__ import annotations

import os
import sys
import time
import logging
from threading import Lock
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

logger = logging.getLogger(__name__)

_PROFILE_REQUEST_TIMING = "VEENA3_TIMING_LOGS"
_inflight_counter = 0
_inflight_lock = Lock()

def _timing_enabled() -> bool:
    """Return True when detailed request timing should be emitted."""
    return os.getenv(_PROFILE_REQUEST_TIMING, "").lower() in {"1", "true", "yes", "on"}

def _with_inflight_delta(delta: int) -> int:
    """Update in-flight request counter and return the new value."""
    global _inflight_counter
    with _inflight_lock:
        _inflight_counter = max(0, _inflight_counter + delta)
        return _inflight_counter

@dataclass
class TTSRuntime:
    """
    Holds long-lived, per-container inference objects.

    Thread Safety:
        This class is designed for async concurrency within a single container.
        The vLLM engine handles internal batching and scheduling.
        Do NOT share across processes without proper synchronization.
    """
    model: Any = None
    pipeline: Any = None
    streaming_pipeline: Any = None
    prompt_builder: Any = None
    bicodec_decoder: Any = None
    sr_service: Optional[Any] = None

    model_version: str = "not_loaded"
    is_loaded: bool = False
    load_time_ms: float = 0.0

    # Configuration
    model_path: str = ""
    bicodec_path: str = ""
    sr_checkpoint_dir: Optional[str] = None
    device: str = "cuda"

    # OPTIMIZATION: Pre-computed global tokens per speaker (eliminates ~110ms pre-roll in streaming)
    # Maps speaker_name -> list of 32 global token IDs captured at startup
    speaker_global_cache: Dict[str, List[int]] = field(default_factory=dict)

# Module-level singleton (per-container)
_runtime: Optional[TTSRuntime] = None

def get_runtime() -> Optional[TTSRuntime]:
    """Get the current TTS runtime singleton."""
    return _runtime

def is_initialized() -> bool:
    """Check if the runtime is initialized and ready."""
    return _runtime is not None and _runtime.is_loaded

def initialize_runtime(
    model_path: Optional[str] = None,
    bicodec_path: Optional[str] = None,
    sr_checkpoint_dir: Optional[str] = None,
    device: str = "cuda",
    hf_token: Optional[str] = None,
    gpu_memory_utilization: float = 0.25,
    enable_sr: bool = False,
    num_engines: int = 1,
) -> TTSRuntime:
    """
    Initialize the TTS runtime with all components.

    This should be called once per container (e.g., in Modal's @modal.enter).

    Args:
        model_path: Path to Spark TTS model (env: SPARK_TTS_MODEL_PATH)
        bicodec_path: Path to BiCodec model (env: BICODEC_MODEL_PATH, defaults to model_path)
        sr_checkpoint_dir: Path to super-resolution checkpoints (env: AP_BWE_CHECKPOINT_DIR)
        device: Device for inference (cuda/cpu)
        hf_token: HuggingFace token for private models (env: HF_TOKEN)
        gpu_memory_utilization: vLLM GPU memory fraction (default: 0.25)
        enable_sr: Enable super-resolution service
        num_engines: Number of vLLM engine instances (default: 1, set to 2-3 for Tier 3 optimization)

    Returns:
        Initialized TTSRuntime instance

    Raises:
        RuntimeError: If initialization fails
    """
    global _runtime

    start_time = time.time()

    # Resolve paths from env vars if not provided
    model_path = model_path or os.environ.get(
        'SPARK_TTS_MODEL_PATH',
        os.environ.get('MODEL_PATH', '/models/spark_tts_4speaker')
    )
    bicodec_path = bicodec_path or os.environ.get(
        'BICODEC_MODEL_PATH',
        model_path  # BiCodec is usually in the same directory
    )
    sr_checkpoint_dir = sr_checkpoint_dir or os.environ.get('AP_BWE_CHECKPOINT_DIR')
    hf_token = hf_token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')

    logger.info(f"Initializing TTS runtime...")
    logger.info(f"  Model path: {model_path}")
    logger.info(f"  BiCodec path: {bicodec_path}")
    logger.info(f"  SR enabled: {enable_sr}, path: {sr_checkpoint_dir}")

    try:
        # Import inference components (vendored into veena3modal for Modal deployment)
        from veena3modal.core.model_loader import SparkTTSModel
        from veena3modal.core.pipeline import SparkTTSPipeline
        from veena3modal.core.bicodec_decoder import BiCodecDecoder
        from veena3modal.processing.prompt_builder import IndicPromptBuilder

        # Load SparkTTS model with vLLM engine
        if num_engines > 1:
            # TIER 3 OPTIMIZATION: Multiple vLLM engines on same GPU
            # Each engine gets gpu_memory_utilization / num_engines to share GPU fairly.
            # E.g., 3 engines at 0.08 each = 0.24 total, ~8GB per engine, ~24GB total.
            from veena3modal.core.multi_engine import create_multi_engine_model
            per_engine_mem = gpu_memory_utilization / num_engines
            logger.info(f"Loading {num_engines} vLLM engines ({per_engine_mem:.2f} GPU mem each)...")
            model = create_multi_engine_model(
                model_path=model_path,
                num_engines=num_engines,
                hf_token=hf_token,
                gpu_memory_per_engine=per_engine_mem,
            )
        else:
            logger.info("Loading SparkTTS model with vLLM...")
            model = SparkTTSModel(
                model_path=model_path,
                hf_token=hf_token,
                gpu_memory_utilization=gpu_memory_utilization,
            )

        # Initialize prompt builder
        logger.info("Initializing prompt builder...")
        prompt_builder = IndicPromptBuilder(
            tokenizer=model.tokenizer,
            model=model,
        )

        # Initialize BiCodec decoder


Summary: auto


rg --line-number "def generate_speech_indic|return_timing" veena3modal/core/pipeline.py


Output:
Chunk ID: 868b83
Wall time: 0.0523 seconds
Process exited with code 0
Original token count: 178
Output:
78:        return_timing: bool = False,
95:            return_timing: If true, returns (audio_bytes, metrics_dict)
99:            If return_timing is true, returns tuple(audio_bytes, metrics).
121:        emit_timing = return_timing or _detailed_timing_enabled()
196:            if return_timing:
239:            if return_timing:
256:            if return_timing:
277:            if return_timing:
317:        if return_timing:
321:    async def generate_speech_indic(
331:        return_timing: bool = False,
352:            return_timing: If true, returns (audio_bytes, metrics_dict)
356:            If return_timing is true, returns tuple(audio_bytes, metrics).
368:            return_timing=return_timing,


I found timing hooks already present in both runtime and pipeline, so I’ll align non-streaming and chunked paths with those hooks, then extend chunking internals to aggregate per-batch/per-decode timing into a single request-level metrics object used by stress tests.


sed -n '300,420p' veena3modal/core/pipeline.py


Output:
Chunk ID: 6c139c
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 999
Output:
            "llm_gpu_batch_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_p50": int(median(batch_gpu_ms)) if batch_gpu_ms else 0,
            "decode_cpu_ms": int(decode_ms),
            "parse_ms": int(parse_ms),
            "total_ms": int(total_ms),
        }
        if token_batches:
            timing_metrics["tokens_per_batch_min"] = min(token_batches)
            timing_metrics["tokens_per_batch_max"] = max(token_batches)
            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))

        if emit_timing:
            logger.info("tts_llm_complete", extra=timing_metrics)

        from veena3modal.audio.utils import add_wav_header
        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)

        if return_timing:
            return wav_bytes, timing_metrics
        return wav_bytes

    async def generate_speech_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ):
        """
        Generate speech audio for Spark TTS (speaker-based).

        This method is kept for backward compatibility with existing API.
        It delegates to generate_speech() with the same implementation.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "Hello <laugh> this is fun!" (will be normalized)
                - "नमस्ते! [excited] आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max BiCodec tokens to generate
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Delegate to generate_speech() with the same implementation for Spark TTS
        return await self.generate_speech(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
            return_timing=return_timing,
            request_id=request_id,
        )

    def _extract_bicodec_tokens(self, generated_text: str) -> Tuple[List[int], List[int]]:
        """
        Extract BiCodec semantic and global tokens from generated text using regex.

        Spark TTS generates tokens in the format:
        - <|bicodec_semantic_{id}|>
        - <|bicodec_global_{id}|>

        Args:
            generated_text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
            Returns ([], []) if no tokens found
        """
        # Extract semantic tokens
        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", generated_text)
        semantic_ids = [int(t) for t in semantic_matches]

        # Extract global tokens
        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
        global_ids = [int(t) for t in global_matches]

        # Only log if there's an issue
        if not semantic_ids and not global_ids:
            logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")

        return semantic_ids, global_ids


sed -n '40,190p' veena3modal/core/pipeline.py


Output:
Chunk ID: e3b28d
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 1360
Output:

class SparkTTSPipeline:
    """
    End-to-end TTS pipeline for Spark TTS with BiCodec.

    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        bicodec_decoder,
    ):
        """
        Initialize pipeline.

        Args:
            model: SparkTTS Model instance
            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
            bicodec_decoder: BiCodecDecoder instance
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.bicodec_decoder = bicodec_decoder

        print(f"🚀 SparkTTSPipeline initialized")

    async def generate_speech(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
        """
        Generate speech audio (non-streaming) using Spark TTS.

        NOTE: This method signature changed from description-based to speaker-based
        to align with Spark TTS architecture.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize (with optional [emotion] tags)
            temperature: Sampling temperature (default: 0.8 for Spark TTS)
            top_k: Top-k sampling (default: 50)
            top_p: Nucleus sampling (default: 1.0)
            max_tokens: Max BiCodec tokens to generate (default: 2048)
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Build prompt using Spark TTS format
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling for Spark TTS
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
        )

        # Use vLLM engine to generate
        if request_id is None:
            import uuid
            request_id = f"req-{uuid.uuid4().hex[:12]}"

        emit_timing = return_timing or _detailed_timing_enabled()
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Collect all outputs and track generation timing
        final_output = None
        token_batches: List[int] = []
        batch_wall_ms: List[float] = []
        batch_gpu_ms: List[float] = []
        total_token_ids = 0
        first_token_ms: Optional[float] = None
        t_start = asyncio.get_event_loop().time()

        iterator = results_generator.__aiter__()
        while True:
            wall_start = asyncio.get_event_loop().time()
            start_event = None
            end_event = None
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                start_event = torch.cuda.Event(enable_timing=True)
                end_event = torch.cuda.Event(enable_timing=True)
                start_event.record()

            try:
                request_output = await iterator.__anext__()
            except StopAsyncIteration:
                break

            wall_end = asyncio.get_event_loop().time()
            batch_ms = (wall_end - wall_start) * 1000
            batch_wall_ms.append(batch_ms)

            batch_gpu_ms_value: Optional[float] = None
            if start_event is not None and end_event is not None:
                end_event.record()
                torch.cuda.synchronize()
                batch_gpu_ms_value = start_event.elapsed_time(end_event)
            if batch_gpu_ms_value is not None:
                batch_gpu_ms.append(batch_gpu_ms_value)

            total_generated = len(request_output.outputs[0].token_ids)
            new_tokens = max(0, total_generated - total_token_ids)
            total_token_ids = total_generated
            token_batches.append(new_tokens)

            if first_token_ms is None:
                first_token_ms = (wall_end - t_start) * 1000

            if emit_timing:
                logger.info(
                    "tts_llm_batch",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "batch_index": len(token_batches),
                        "batch_tokens": new_tokens,
                        "llm_batch_wall_ms": round(batch_ms, 3),
                        "llm_batch_gpu_ms": round(batch_gpu_ms_value, 3)
                        if batch_gpu_ms_value is not None
                        else None,
                    },
                )
            final_output = request_output

        if final_output is None:
            if emit_timing:


sed -n '1,320p' scripts/stress_test_production.py


Output:
Chunk ID: 49a229
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 3595
Output:
#!/usr/bin/env python3
"""
Production Stress Test - Real-world sentences for Veena3 TTS.

Uses actual production-quality sentences across:
- Multiple languages (English, Hindi, Telugu)
- Various lengths (short greetings to long paragraphs)
- Emotion tags
- Mixed scripts
- Numbers, dates, special characters

Runs the same concurrency ramp as the baseline test for direct comparison.
"""

import argparse
import asyncio
import json
import os
import statistics
import subprocess
import sys
import time
import threading
from dataclasses import dataclass, field
from typing import List, Optional, Dict, Any

import httpx

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

# === Real production sentences ===
# These mimic actual API traffic: mixed languages, emotions, varied lengths

REAL_SENTENCES = {
    "en_short": [
        "Welcome to Veena, your personal voice assistant.",
        "The weather today is sunny with a high of thirty two degrees.",
        "Your order has been confirmed and will arrive by tomorrow.",
        "Please hold on, I am transferring you to a specialist.",
        "Thank you for calling. Have a wonderful day!",
    ],
    "en_medium": [
        "Good morning! I hope you are having a great start to your day. Let me walk you through the key highlights from yesterday's meeting.",
        "The quarterly revenue report shows a fifteen percent increase compared to last year, driven primarily by growth in the South Asian market.",
        "I would like to remind you that your subscription is expiring on March fifteenth. Would you like me to help you with the renewal process?",
        "According to the latest research, artificial intelligence is expected to contribute over fifteen trillion dollars to the global economy by twenty thirty.",
        "[excited] Congratulations! You have been selected for our premium membership program. Let me tell you about the amazing benefits.",
    ],
    "en_long": [
        "In the heart of Hyderabad, where the old city meets the new, there is a small bookshop that has been running for over fifty years. The owner, Ramesh uncle, knows every book by heart. He can tell you exactly which shelf to find your favorite author on, and he always has a recommendation ready. Last Tuesday, a young girl walked in looking for a book on astronomy. Ramesh uncle smiled and led her to the science section.",
        "The Indian space program has achieved remarkable milestones in the past decade. From the Mars Orbiter Mission in twenty fourteen to the Chandrayaan three lunar landing in twenty twenty three, ISRO has consistently demonstrated that world class space exploration does not require a massive budget. The upcoming Gaganyaan mission will mark India's first crewed spaceflight.",
    ],
    "hi_short": [
        "नमस्ते! आज का दिन बहुत अच्छा है।",
        "कृपया अपना नाम और फोन नंबर बताइए।",
        "आपका ऑर्डर सफलतापूर्वक प्लेस हो गया है।",
        "धन्यवाद! आपकी कॉल हमारे लिए महत्वपूर्ण है।",
    ],
    "hi_medium": [
        "[curious] क्या आप जानते हैं कि भारत दुनिया का सबसे बड़ा लोकतंत्र है? यहाँ एक सौ चालीस करोड़ से अधिक लोग रहते हैं।",
        "आज के मौसम का हाल बताते हैं। दिल्ली में तापमान बत्तीस डिग्री रहेगा और हल्की बारिश की संभावना है।",
    ],
    "te_short": [
        "నమస్కారం! మీరు ఎలా ఉన్నారు?",
        "మీ ఆర్డర్ విజయవంతంగా ప్లేస్ చేయబడింది।",
        "దయచేసి కొంచెం ఆగండి, మేము మీకు సహాయం చేస్తాము.",
    ],
    "emotion_mixed": [
        "[laughs] Oh that is so funny! I cannot believe you said that.",
        "[whispers] Let me tell you a secret. The new product launch is happening next week.",
        "[excited] Amazing news everyone! We just crossed one million users!",
        "[sighs] Unfortunately, the flight has been delayed by two hours. I apologize for the inconvenience.",
        "[angry] This is unacceptable! The service was supposed to be restored by noon today.",
    ],
}

# Flatten and tag for round-robin selection
ALL_SENTENCES = []
for category, sentences in REAL_SENTENCES.items():
    for s in sentences:
        ALL_SENTENCES.append({"text": s, "category": category, "length": len(s)})

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika",
            "adarsh", "Nilay", "Aarvi", "Asha", "Bittu", "Mira"]

# === GPU Monitor (reused from stress_test_local.py) ===

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=5)

    def _poll(self):
        while self._running:
            try:
                r = subprocess.run(
                    ["nvidia-smi", "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                     "--format=csv,noheader,nounits"],
                    capture_output=True, text=True, timeout=5)
                if r.returncode == 0:
                    parts = r.stdout.strip().split(", ")
                    if len(parts) >= 4:
                        self.snapshots.append(GPUSnapshot(
                            timestamp=time.time(),
                            memory_used_mb=float(parts[0]),
                            memory_total_mb=float(parts[1]),
                            gpu_utilization_pct=float(parts[2]),
                            temperature_c=float(parts[3]),
                        ))
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self):
        if not self.snapshots:
            return {}
        mem = [s.memory_used_mb for s in self.snapshots]
        util = [s.gpu_utilization_pct for s in self.snapshots]
        return {
            "memory_mb_avg": statistics.mean(mem),
            "memory_mb_peak": max(mem),
            "memory_mb_total": self.snapshots[0].memory_total_mb,
            "gpu_util_avg": statistics.mean(util),
            "gpu_util_peak": max(util),
            "temp_avg": statistics.mean(s.temperature_c for s in self.snapshots),
            "temp_peak": max(s.temperature_c for s in self.snapshots),
        }

# === Request Result ===

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float
    ttfb_ms: float = 0.0
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    error: Optional[str] = None
    text_length: int = 0
    category: str = ""
    stream: bool = False

# === HTTP Client ===

async def make_request(client, base_url, text, speaker, stream=False, timeout=120.0):
    url = f"{base_url}{GENERATE_PATH}"
    payload = {"text": text, "speaker": speaker, "stream": stream, "format": "wav"}
    start = time.time()

    try:
        if stream:
            async with client.stream("POST", url, json=payload,
                                     headers={"Content-Type": "application/json"},
                                     timeout=timeout) as resp:
                first_byte = True
                ttfb = 0.0
                total = 0
                async for chunk in resp.aiter_bytes():
                    if first_byte:
                        ttfb = (time.time() - start) * 1000
                        first_byte = False
                    total += len(chunk)
                latency = (time.time() - start) * 1000
                if resp.status_code == 200:
                    pcm = max(0, total - 44)
                    return RequestResult(True, 200, latency, ttfb, total,
                                        pcm / (16000 * 2), text_length=len(text), stream=True)
                return RequestResult(False, resp.status_code, latency, ttfb,
                                     error=f"HTTP {resp.status_code}", text_length=len(text), stream=True)
        else:
            resp = await client.post(url, json=payload,
                                     headers={"Content-Type": "application/json"},
                                     timeout=timeout)
            latency = (time.time() - start) * 1000
            audio_sec = 0.0
            h = resp.headers.get("X-Audio-Seconds")
            if h:
                audio_sec = float(h)
            elif resp.status_code == 200:
                audio_sec = max(0, len(resp.content) - 44) / (16000 * 2)
            if resp.status_code == 200:
                return RequestResult(True, 200, latency, latency, len(resp.content),
                                     audio_sec, text_length=len(text), stream=False)
            return RequestResult(False, resp.status_code, latency,
                                 error=resp.text[:200], text_length=len(text), stream=False)
    except httpx.TimeoutException:
        return RequestResult(False, 0, (time.time() - start) * 1000, error="TIMEOUT",
                             text_length=len(text), stream=stream)
    except Exception as e:
        return RequestResult(False, 0, (time.time() - start) * 1000, error=str(e)[:200],
                             text_length=len(text), stream=stream)

def percentile(values, pct):
    if not values:
        return 0
    s = sorted(values)
    return s[min(int(len(s) * pct / 100), len(s) - 1)]

async def run_level(base_url, num_requests, concurrency, stream, gpu_monitor=None):
    sem = asyncio.Semaphore(concurrency)
    results = []

    async def req(i, client):
        async with sem:
            item = ALL_SENTENCES[i % len(ALL_SENTENCES)]
            speaker = SPEAKERS[i % len(SPEAKERS)]
            r = await make_request(client, base_url, item["text"], speaker, stream)
            r.category = item["category"]
            return r

    if gpu_monitor:
        gpu_monitor.snapshots.clear()

    start = time.time()
    limits = httpx.Limits(max_connections=concurrency + 10, max_keepalive_connections=concurrency)
    async with httpx.AsyncClient(limits=limits) as client:
        tasks = [req(i, client) for i in range(num_requests)]
        raw = await asyncio.gather(*tasks, return_exceptions=True)
    wall = time.time() - start

    for r in raw:
        if isinstance(r, Exception):
            results.append(RequestResult(False, 0, 0, error=str(r)[:200], stream=stream))
        else:
            results.append(r)

    successes = [r for r in results if r.success]
    failures = [r for r in results if not r.success]
    lats = [r.latency_ms for r in successes]
    ttfbs = [r.ttfb_ms for r in successes if r.ttfb_ms > 0]
    audio_total = sum(r.audio_seconds for r in successes)
    rps = num_requests / wall if wall > 0 else 0
    rtf = wall / audio_total if audio_total > 0 else 0

    gpu = gpu_monitor.summary() if gpu_monitor else {}
    mode = "streaming" if stream else "non-streaming"
    label = f"{concurrency}c-production-{mode}"

    print(f"\n{'=' * 72}")
    print(f"  {label.upper()}")
    print(f"  {num_requests} requests, {concurrency} concurrent, {mode}")
    print(f"  Sentences: real production ({len(ALL_SENTENCES)} unique, multilingual)")
    print(f"{'=' * 72}")
    print(f"  Success:     {len(successes)}/{num_requests} ({len(successes)/num_requests:.0%})")
    print(f"  Failures:    {len(failures)}")
    print(f"  Wall time:   {wall:.2f}s")
    print(f"  Throughput:  {rps:.2f} req/s")
    print(f"  Audio total: {audio_total:.1f}s")
    print(f"  Eff. RTF:    {rtf:.3f}")

    if lats:
        print(f"\n  Latency (ms):")
        print(f"    min={min(lats):.0f}  avg={statistics.mean(lats):.0f}  "
              f"p50={percentile(lats,50):.0f}  p95={percentile(lats,95):.0f}  "
              f"p99={percentile(lats,99):.0f}  max={max(lats):.0f}")

    if ttfbs and stream:
        print(f"  TTFB (ms):")
        print(f"    min={min(ttfbs):.0f}  avg={statistics.mean(ttfbs):.0f}  "
              f"p50={percentile(ttfbs,50):.0f}  p95={percentile(ttfbs,95):.0f}  max={max(ttfbs):.0f}")

    if gpu:
        print(f"\n  GPU:")
        print(f"    Memory: {gpu['memory_mb_avg']:.0f}MB avg, {gpu['memory_mb_peak']:.0f}MB peak / {gpu['memory_mb_total']:.0f}MB")
        print(f"    Util:   {gpu['gpu_util_avg']:.0f}% avg, {gpu['gpu_util_peak']:.0f}% peak")
        print(f"    Temp:   {gpu['temp_avg']:.0f}C avg, {gpu['temp_peak']:.0f}C peak")

    errors = {}
    for r in failures:
        k = (r.error or f"HTTP {r.status_code}")[:60]
        errors[k] = errors.get(k, 0) + 1
    if errors:
        print(f"\n  Errors:")
        for e, c in sorted(errors.items(), key=lambda x: -x[1]):
            print(f"    [{c}x] {e}")

    return {
        "level": label, "concurrency": concurrency, "requests": num_requests,
        "mode": mode, "success_rate": len(successes) / num_requests if num_requests else 0,
        "throughput_rps": rps, "wall_time_s": wall, "eff_rtf": rtf,
        "audio_total_s": audio_total,
        "latency_ms": {"avg": statistics.mean(lats), "p50": percentile(lats, 50),
                        "p95": percentile(lats, 95), "p99": percentile(lats, 99),
                        "max": max(lats)} if lats else {},
        "ttfb_ms": {"avg": statistics.mean(ttfbs), "p50": percentile(ttfbs, 50),
                     "p95": percentile(ttfbs, 95), "max": max(ttfbs)} if ttfbs else {},
        "gpu": gpu, "errors": errors,
    }

async def main_async(args):
    base_url = args.url


rg --line-number "timing_metrics|llm_batch_tokens|llm_batch_wall_ms" veena3modal/core/pipeline.py && sed -n '200,340p' veena3modal/core/pipeline.py


Output:
Chunk ID: bfa13b
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 1539
Output:
181:                        "llm_batch_wall_ms": round(batch_ms, 3),
288:        timing_metrics: Dict[str, Any] = {
307:            timing_metrics["tokens_per_batch_min"] = min(token_batches)
308:            timing_metrics["tokens_per_batch_max"] = max(token_batches)
309:            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))
312:            logger.info("tts_llm_complete", extra=timing_metrics)
318:            return wav_bytes, timing_metrics
                    "token_total": 0,
                    "batch_count": 0,
                }
            return None

        # Extract generated text (contains BiCodec tokens)
        parse_start = asyncio.get_event_loop().time()
        generated_text = final_output.outputs[0].text
        generated_ids = final_output.outputs[0].token_ids

        # Extract BiCodec tokens using regex
        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)

        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000

        if not semantic_ids or not global_ids:
            if emit_timing:
                logger.error(
                    "tts_bicodec_parse_failed",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "semantic_count": len(semantic_ids),
                        "global_count": len(global_ids),
                        "token_total": len(generated_ids),
                    },
                )
            logger.error(
                "❌ No BiCodec tokens found in output",
                extra={
                    "speaker": speaker,
                    "text_length": len(text),
                    "text_preview": text[:100],
                    "generated_tokens": len(generated_ids),
                    "semantic_tokens": len(semantic_ids),
                    "global_tokens": len(global_ids),
                    "generated_text_preview": generated_text[:500],
                }
            )
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Validate tokens
        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
            logger.error("❌ BiCodec token validation failed", extra={
                "speaker": speaker,
                "text_length": len(text),
                "semantic_tokens": len(semantic_ids),
                "global_tokens": len(global_ids),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Decode to audio
        decode_start = asyncio.get_event_loop().time()
        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000

        if audio_bytes is None:
            if emit_timing:
                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
            logger.error("❌ BiCodec decode failed", extra={
                "speaker": speaker,
                "text_length": len(text),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        total_ms = (asyncio.get_event_loop().time() - t_start) * 1000

        timing_metrics: Dict[str, Any] = {
            "request_id": request_id,
            "speaker": speaker,
            "text_length": len(text),
            "batch_count": len(token_batches),
            "token_total": len(generated_ids),
            "token_batches": token_batches,
            "ttfb_ms": int(first_token_ms or 0),
            "llm_wall_ms": int(sum(batch_wall_ms)),
            "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
            "llm_gpu_batch_ms_total": int(sum(batch_gpu_ms)),
            "llm_gpu_batch_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_p50": int(median(batch_gpu_ms)) if batch_gpu_ms else 0,
            "decode_cpu_ms": int(decode_ms),
            "parse_ms": int(parse_ms),
            "total_ms": int(total_ms),
        }
        if token_batches:
            timing_metrics["tokens_per_batch_min"] = min(token_batches)
            timing_metrics["tokens_per_batch_max"] = max(token_batches)
            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))

        if emit_timing:
            logger.info("tts_llm_complete", extra=timing_metrics)

        from veena3modal.audio.utils import add_wav_header
        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)

        if return_timing:
            return wav_bytes, timing_metrics
        return wav_bytes

    async def generate_speech_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ):
        """
        Generate speech audio for Spark TTS (speaker-based).

        This method is kept for backward compatibility with existing API.
        It delegates to generate_speech() with the same implementation.

        Args:


nl -ba veena3modal/core/streaming_pipeline.py | sed -n '560,1100p'


Output:
Chunk ID: 500ef1
Wall time: 0.0533 seconds
Process exited with code 0
Original token count: 6075
Output:
   560	                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
   561	                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
   562	                "llm_parse_ms": int(sum(parse_wall_ms_list)),
   563	                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
   564	                "decode_calls": decode_calls,
   565	                "llm_decode_wall_ms_total": int(decode_wall_ms),
   566	                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
   567	                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
   568	                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
   569	                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
   570	                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
   571	                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
   572	                "ttfb_ms": int(ttfb_ms),
   573	                "generation_ms": int(total_wall_ms),
   574	                "audio_duration_seconds": audio_duration_s,
   575	                "audio_chunks": total_audio_chunks,
   576	                "samples_emitted": total_samples_emitted_to_user,
   577	                "text_chunked": False,
   578	                "concurrency_mode": "single",
   579	            }
   580	        )
   581	        if emit_timing:
   582	            logger.info("tts_stream_indic_complete", extra=request_metrics)
   583
   584	    async def generate_speech_stream_indic_first_chunk(
   585	        self,
   586	        speaker: str,
   587	        text: str,
   588	        temperature: float = DEFAULT_TEMPERATURE,
   589	        top_k: int = DEFAULT_TOP_K,
   590	        top_p: float = DEFAULT_TOP_P,
   591	        max_tokens: int = DEFAULT_MAX_TOKENS,
   592	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   593	        seed: Optional[int] = None,
   594	        request_id: Optional[str] = None,
   595	        request_metrics: Optional[Dict[str, Any]] = None,
   596	    ) -> AsyncGenerator[tuple, None]:
   597	        """
   598	        Generate speech for the FIRST chunk of multi-chunk text, capturing global tokens.
   599
   600	        This method is specifically for chunked generation: it yields (audio_bytes, global_ids)
   601	        tuples instead of just audio_bytes. The caller captures global_ids from the first
   602	        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.
   603
   604	        Use Case (chunked streaming with voice consistency):
   605	            globals_captured = None
   606	            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
   607	                if globals_captured is None and global_ids:
   608	                    globals_captured = global_ids  # Capture once
   609	                yield audio_bytes
   610	            # Now use globals_captured for subsequent chunks
   611
   612	        Args:
   613	            speaker: Speaker name
   614	            text: First text chunk to synthesize
   615	            temperature: Sampling temperature
   616	            top_k: Top-k sampling
   617	            top_p: Nucleus sampling
   618	            max_tokens: Max tokens to generate
   619	            repetition_penalty: Prevent repetition
   620	            seed: Random seed for reproducibility
   621
   622	        Yields:
   623	            Tuple of (audio_bytes, global_ids)
   624	            - audio_bytes: Raw PCM audio (int16, 16kHz)
   625	            - global_ids: List of 32 global token IDs (populated after first decode, else empty)
   626
   627	        Thread Safety:
   628	            This method is stateless and thread-safe. Each call creates its own
   629	            request-scoped state. No global state is modified or shared.
   630	        """
   631	        import time
   632	        t_start = time.time()
   633	        request_metrics = request_metrics or {}
   634	        request_metrics.setdefault("request_id", request_id)
   635
   636	        # Build prompt using Indic prompt builder
   637	        prompt = self.prompt_builder.build_prefix(speaker, text)
   638
   639	        # Configure sampling
   640	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
   641	        sampling_params = SamplingParams(
   642	            temperature=temperature,
   643	            top_k=top_k,
   644	            top_p=top_p,
   645	            max_tokens=max_tokens,
   646	            min_tokens=DEFAULT_MIN_TOKENS,
   647	            repetition_penalty=repetition_penalty,
   648	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
   649	            skip_special_tokens=False,
   650	            seed=seed,
   651	        )
   652
   653	        # BiCodec token buffers
   654	        semantic_buffer = []
   655	        global_buffer = []
   656	        processed_token_count = 0  # Track how many vLLM tokens we've parsed
   657	        total_audio_chunks = 0
   658
   659	        # BiCodec streaming configuration
   660	        EXPECTED_GLOBAL_COUNT = 32
   661	        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
   662	        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
   663	        CROSSFADE_MS = 50
   664
   665	        # Generate unique request ID
   666	        import uuid
   667	        if request_id is None:
   668	            request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
   669	            request_metrics["request_id"] = request_id
   670
   671	        # Reuse pre-warmed token parser (singleton, created once in __init__)
   672	        token_parser = self.token_parser
   673	        if token_parser is None:
   674	            raise RuntimeError("BiCodecTokenParser unavailable for first-chunk streaming")
   675
   676	        results_generator = self.model.engine.generate(
   677	            prompt=prompt,
   678	            sampling_params=sampling_params,
   679	            request_id=request_id,
   680	        )
   681
   682	        # Track state
   683	        last_decode_count = 0
   684	        total_samples_emitted_to_user = 0
   685	        previous_chunk_tail: Optional[bytes] = None
   686	        captured_globals: List[int] = []  # Will be populated once we have 32 globals
   687
   688	        # Stream tokens with BiCodec extraction
   689	        async for request_output in results_generator:
   690	            generated_ids = request_output.outputs[0].token_ids
   691
   692	            # OPTIMIZATION: Incremental parsing (O(1) per new token)
   693	            new_token_ids = generated_ids[processed_token_count:]
   694	            processed_token_count = len(generated_ids)
   695	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
   696
   697	            # Capture global tokens once we have all 32
   698	            if not captured_globals and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   699	                captured_globals = global_buffer[:EXPECTED_GLOBAL_COUNT]
   700
   701	            # Process if we have all global tokens
   702	            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   703	                semantic_count = len(semantic_buffer)
   704	                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   705	                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
   706	                        last_decode_count = semantic_count
   707
   708	                        all_semantic = semantic_buffer
   709	                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]
   710
   711	                        if self.snac_decoder.enable_batching:
   712	                            audio_bytes = await self.snac_decoder.decode_single_async(
   713	                                semantic_ids=all_semantic,
   714	                                global_ids=window_global,
   715	                                trim_warmup=False,
   716	                                use_sliding_window=False
   717	                            )
   718	                        else:
   719	                            audio_bytes = self.snac_decoder.decode_streaming(
   720	                                semantic_ids=all_semantic,
   721	                                global_ids=window_global,
   722	                                use_sliding_window=False,
   723	                                trim_warmup=False
   724	                            )
   725
   726	                        if audio_bytes:
   727	                            total_samples_decoded = len(audio_bytes) // 2
   728
   729	                            if total_samples_decoded > total_samples_emitted_to_user:
   730	                                new_bytes_start = total_samples_emitted_to_user * 2
   731	                                new_audio_bytes = audio_bytes[new_bytes_start:]
   732
   733	                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   734	                                    previous_chunk_tail,
   735	                                    new_audio_bytes,
   736	                                    sample_rate_hz=16000,
   737	                                    crossfade_ms=CROSSFADE_MS,
   738	                                )
   739
   740	                                samples_emitted_in_this_chunk = len(to_emit) // 2
   741	                                total_samples_emitted_to_user += samples_emitted_in_this_chunk
   742
   743	                                if to_emit:
   744	                                    total_audio_chunks += 1
   745	                                    # Yield tuple: (audio_bytes, global_ids)
   746	                                    yield (to_emit, captured_globals)
   747
   748	        # Final chunk processing
   749	        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   750	            audio_bytes = self.snac_decoder.decode_streaming(
   751	                semantic_ids=semantic_buffer,
   752	                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
   753	                use_sliding_window=False,
   754	                trim_warmup=False
   755	            )
   756	            if audio_bytes:
   757	                total_samples_decoded = len(audio_bytes) // 2
   758
   759	                if total_samples_decoded > total_samples_emitted_to_user:
   760	                    new_bytes_start = total_samples_emitted_to_user * 2
   761	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   762
   763	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   764	                        previous_chunk_tail,
   765	                        new_audio_bytes,
   766	                        sample_rate_hz=16000,
   767	                        crossfade_ms=CROSSFADE_MS,
   768	                    )
   769
   770	                    samples_emitted_in_final = len(to_emit) // 2
   771	                    total_samples_emitted_to_user += samples_emitted_in_final
   772
   773	                        if to_emit:
   774	                            total_audio_chunks += 1
   775	                            yield (to_emit, captured_globals)
   776
   777	        # Flush remaining tail
   778	        if previous_chunk_tail:
   779	            total_audio_chunks += 1
   780	            tail_samples = len(previous_chunk_tail) // 2
   781	            total_samples_emitted_to_user += tail_samples
   782	            yield (previous_chunk_tail, captured_globals)
   783
   784	        t_end = time.time()
   785	        audio_duration_s = total_samples_emitted_to_user / 16000
   786	        request_metrics.update(
   787	            {
   788	                "request_id": request_id,
   789	                "speaker": speaker,
   790	                "audio_duration_seconds": audio_duration_s,
   791	                "audio_chunks": total_audio_chunks,
   792	                "text_chunked": True,
   793	                "captured_global_count": len(captured_globals),
   794	                "semantic_tokens": len(semantic_buffer),
   795	                "generation_ms": int((t_end - t_start) * 1000),
   796	            }
   797	        )
   798	        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)
   799
   800	    async def generate_speech_stream_indic_continuation(
   801	        self,
   802	        speaker: str,
   803	        text: str,
   804	        global_ids: List[int],
   805	        temperature: float = DEFAULT_TEMPERATURE,
   806	        top_k: int = DEFAULT_TOP_K,
   807	        top_p: float = DEFAULT_TOP_P,
   808	        max_tokens: int = DEFAULT_MAX_TOKENS,
   809	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   810	        seed: Optional[int] = None,
   811	        request_id: Optional[str] = None,
   812	        request_metrics: Optional[Dict[str, Any]] = None,
   813	    ) -> AsyncGenerator[bytes, None]:
   814	        """
   815	        Generate speech for CONTINUATION chunks using pre-captured global tokens.
   816
   817	        This is the key method for voice consistency in chunked generation:
   818	        - Uses global_ids from the first chunk to maintain identical voice
   819	        - Model generates only semantic tokens (skips global token generation)
   820	        - Ensures no voice drift across chunks
   821
   822	        CRITICAL for production:
   823	        - global_ids is request-scoped, passed explicitly (no shared state)
   824	        - Thread-safe: each request has its own state
   825	        - No global caching that could cause cross-request contamination
   826
   827	        Args:
   828	            speaker: Speaker name (must match first chunk)
   829	            text: Continuation text chunk to synthesize
   830	            global_ids: 32 global token IDs captured from first chunk
   831	            temperature: Sampling temperature
   832	            top_k: Top-k sampling
   833	            top_p: Nucleus sampling
   834	            max_tokens: Max tokens to generate
   835	            repetition_penalty: Prevent repetition
   836	            seed: Random seed (use same as first chunk for consistency)
   837
   838	        Yields:
   839	            Audio bytes (int16 PCM, 16kHz mono)
   840
   841	        Raises:
   842	            ValueError: If global_ids doesn't contain exactly 32 tokens
   843
   844	        Thread Safety:
   845	            Fully thread-safe. All state is request-scoped and passed explicitly.
   846	        """
   847	        import time
   848	        t_start = time.time()
   849	        request_metrics = request_metrics or {}
   850	        request_metrics.setdefault("request_id", request_id)
   851
   852	        # Validate global tokens
   853	        EXPECTED_GLOBAL_COUNT = 32
   854	        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
   855	            raise ValueError(
   856	                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
   857	                f"global_ids must be captured from first chunk generation."
   858	            )
   859
   860	        # Build prompt WITH pre-filled global tokens
   861	        # This tells the model to skip global generation and go straight to semantic tokens
   862	        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)
   863
   864	        # Configure sampling
   865	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
   866	        sampling_params = SamplingParams(
   867	            temperature=temperature,
   868	            top_k=top_k,
   869	            top_p=top_p,
   870	            max_tokens=max_tokens,
   871	            min_tokens=DEFAULT_MIN_TOKENS,
   872	            repetition_penalty=repetition_penalty,
   873	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
   874	            skip_special_tokens=False,
   875	            seed=seed,
   876	        )
   877
   878	        # BiCodec token buffers
   879	        semantic_buffer = []
   880	        global_buffer_unused = []  # Not used in continuation, but needed for parser API
   881	        processed_token_count = 0  # Track how many vLLM tokens we've parsed
   882	        total_audio_chunks = 0
   883
   884	        # BiCodec streaming configuration
   885	        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
   886	        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
   887	        CROSSFADE_MS = 50
   888
   889	        # Generate unique request ID
   890	        import uuid
   891	        if request_id is None:
   892	            request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
   893	            request_metrics["request_id"] = request_id
   894
   895	        # Reuse pre-warmed token parser (singleton, created once in __init__)
   896	        token_parser = self.token_parser
   897	        if token_parser is None:
   898	            raise RuntimeError("BiCodecTokenParser unavailable for continuation streaming")
   899
   900	        results_generator = self.model.engine.generate(
   901	            prompt=prompt,
   902	            sampling_params=sampling_params,
   903	            request_id=request_id,
   904	        )
   905
   906	        # Track state
   907	        last_decode_count = 0
   908	        total_samples_emitted_to_user = 0
   909	        previous_chunk_tail: Optional[bytes] = None
   910
   911	        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
   912	        async for request_output in results_generator:
   913	            generated_ids = request_output.outputs[0].token_ids
   914
   915	            # OPTIMIZATION: Incremental parsing (O(1) per new token)
   916	            # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
   917	            new_token_ids = generated_ids[processed_token_count:]
   918	            processed_token_count = len(generated_ids)
   919	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)
   920
   921	            semantic_count = len(semantic_buffer)
   922	            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   923	                if semantic_count - last_decode_count >= DECODE_INTERVAL:
   924	                    last_decode_count = semantic_count
   925
   926	                    # Use pre-captured global tokens for decoding
   927	                    if self.snac_decoder.enable_batching:
   928	                        audio_bytes = await self.snac_decoder.decode_single_async(
   929	                            semantic_ids=semantic_buffer,
   930	                            global_ids=global_ids,  # Use captured globals
   931	                            trim_warmup=False,
   932	                            use_sliding_window=False
   933	                        )
   934	                    else:
   935	                        audio_bytes = self.snac_decoder.decode_streaming(
   936	                            semantic_ids=semantic_buffer,
   937	                            global_ids=global_ids,  # Use captured globals
   938	                            use_sliding_window=False,
   939	                            trim_warmup=False
   940	                        )
   941
   942	                    if audio_bytes:
   943	                        total_samples_decoded = len(audio_bytes) // 2
   944
   945	                        if total_samples_decoded > total_samples_emitted_to_user:
   946	                            new_bytes_start = total_samples_emitted_to_user * 2
   947	                            new_audio_bytes = audio_bytes[new_bytes_start:]
   948
   949	                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
   950	                                previous_chunk_tail,
   951	                                new_audio_bytes,
   952	                                sample_rate_hz=16000,
   953	                                crossfade_ms=CROSSFADE_MS,
   954	                            )
   955
   956	                            samples_emitted_in_this_chunk = len(to_emit) // 2
   957	                            total_samples_emitted_to_user += samples_emitted_in_this_chunk
   958
   959	                        if to_emit:
   960	                            total_audio_chunks += 1
   961	                            yield to_emit
   962
   963	        # Final chunk processing
   964	        if len(semantic_buffer) > last_decode_count:
   965	            audio_bytes = self.snac_decoder.decode_streaming(
   966	                semantic_ids=semantic_buffer,
   967	                global_ids=global_ids,  # Use captured globals
   968	                use_sliding_window=False,
   969	                trim_warmup=False
   970	            )
   971	            if audio_bytes:
   972	                total_samples_decoded = len(audio_bytes) // 2
   973
   974	                if total_samples_decoded > total_samples_emitted_to_user:
   975	                    new_bytes_start = total_samples_emitted_to_user * 2
   976	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   977
   978	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   979	                        previous_chunk_tail,
   980	                        new_audio_bytes,
   981	                        sample_rate_hz=16000,
   982	                        crossfade_ms=CROSSFADE_MS,
   983	                    )
   984
   985	                    samples_emitted_in_final = len(to_emit) // 2
   986	                    total_samples_emitted_to_user += samples_emitted_in_final
   987
   988	                    if to_emit:
   989	                        total_audio_chunks += 1
   990	                        yield to_emit
   991
   992	        # Flush remaining tail
   993	        if previous_chunk_tail:
   994	            total_audio_chunks += 1
   995	            tail_samples = len(previous_chunk_tail) // 2
   996	            total_samples_emitted_to_user += tail_samples
   997	            yield previous_chunk_tail
   998
   999	        t_end = time.time()
  1000	        request_metrics.update(
  1001	            {
  1002	                "request_id": request_id,
  1003	                "speaker": speaker,
  1004	                "audio_duration_seconds": total_samples_emitted_to_user / 16000,
  1005	                "audio_chunks": total_audio_chunks,
  1006	                "semantic_tokens": len(semantic_buffer),
  1007	                "global_tokens": len(global_ids),
  1008	                "generation_ms": int((t_end - t_start) * 1000),
  1009	            }
  1010	        )
  1011	        logger.info("tts_stream_continuation_complete", extra=request_metrics)


nl -ba veena3modal/processing/long_text_processor.py | sed -n '120,280p'


Output:
Chunk ID: 931c08
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1741
Output:
   120	        )
   121
   122	    def should_chunk(self, text: str) -> bool:
   123	        """Determine if text should be chunked."""
   124	        return len(text) > self.CHUNKING_THRESHOLD
   125
   126	    def chunk_text(self, text: str) -> list:
   127	        """
   128	        Chunk text using the internal chunker.
   129
   130	        Delegates to self.chunker.chunk_text() for actual chunking logic.
   131	        This method is used by _stream_chunked_text in tts_runtime.
   132
   133	        Args:
   134	            text: Text to chunk
   135
   136	        Returns:
   137	            List of text chunks
   138	        """
   139	        return self.chunker.chunk_text(text)
   140
   141	    async def generate_with_chunking(
   142	        self,
   143	        text: str,
   144	        speaker: str,
   145	        temperature: float = 0.4,
   146	        top_k: int = 50,  # Added for Spark TTS
   147	        top_p: float = 0.9,
   148	        max_tokens: int = 4096,
   149	        repetition_penalty: float = 1.05,  # Prevent token repetition
   150	        seed: Optional[int] = None,
   151	        sample_rate: int = 16000,  # BiCodec uses 16kHz, not 24kHz
   152	    ) -> Optional[bytes]:
   153	        """
   154	        Generate audio for long text with chunking and stitching.
   155
   156	        Process:
   157	        1. Chunk text at natural boundaries
   158	        2. Generate audio for each chunk
   159	        3. Stitch audio with crossfade
   160
   161	        Args:
   162	            text: Long text to synthesize
   163	            speaker: Speaker name (lipakshi, reet, etc.)
   164	            temperature: Sampling temperature
   165	            top_p: Nucleus sampling
   166	            max_tokens: Max tokens per chunk
   167	            seed: Random seed (same seed = same voice across chunks)
   168	            sample_rate: Audio sample rate
   169
   170	        Returns:
   171	            Stitched audio bytes (int16 PCM) or None if generation fails
   172	        """
   173	        # Detect language mix for logging
   174	        lang_info = self.chunker.detect_language_mix(text)
   175
   176	        logger.info(
   177	            f"Processing long text with chunking",
   178	            extra={
   179	                "text_length": len(text),
   180	                "primary_language": lang_info['primary'],
   181	                "lang_mix": lang_info,
   182	                "speaker": speaker,
   183	            }
   184	        )
   185
   186	        # 1. Chunk text
   187	        text_chunks = self.chunker.chunk_text(text)
   188
   189	        logger.info(
   190	            f"Text chunked into {len(text_chunks)} chunks",
   191	            extra={
   192	                "num_chunks": len(text_chunks),
   193	                "chunk_sizes": [len(chunk) for chunk in text_chunks],
   194	            }
   195	        )
   196
   197	        # 2. Generate audio for each chunk
   198	        audio_chunks = []
   199
   200	        # Log summary at start
   201	        logger.info(
   202	            f"🎵 Generating {len(text_chunks)} chunks for {len(text)} chars",
   203	            extra={
   204	                "total_chunks": len(text_chunks),
   205	                "total_chars": len(text),
   206	                "speaker": speaker,
   207	            }
   208	        )
   209
   210	        for i, chunk in enumerate(text_chunks):
   211	            # Calculate appropriate max_tokens for this chunk
   212	            # BiCodec semantic tokens: ~50 tokens/second, ~5 words/second = ~10 tokens/word
   213	            # Rough estimate: 5 chars/word, so ~2 tokens/char for audio generation
   214	            # Add buffer for safety: multiply by 3 to handle variations
   215	            chunk_max_tokens = min(
   216	                int(len(chunk) * 3),  # ~2 tokens/char * 1.5 safety buffer = 3
   217	                4096  # Cap at model's max context length
   218	            )
   219	            chunk_max_tokens = max(chunk_max_tokens, 100)  # Ensure minimum
   220
   221	            # Generate audio for this chunk (returns WAV format with header)
   222	            audio_bytes = await self.pipeline.generate_speech_indic(
   223	                speaker=speaker,
   224	                text=chunk,
   225	                temperature=temperature,
   226	                top_k=top_k,  # Added for Spark TTS
   227	                top_p=top_p,
   228	                max_tokens=chunk_max_tokens,  # Use calculated limit, not global max_tokens
   229	                repetition_penalty=repetition_penalty,  # Prevent token repetition
   230	                seed=seed,  # Use same seed for consistent voice across chunks
   231	            )
   232
   233	            if audio_bytes is None:
   234	                logger.error(
   235	                    f"❌ Failed to generate audio for chunk {i+1}/{len(text_chunks)}",
   236	                    extra={
   237	                        "chunk_index": i,
   238	                        "chunk_length": len(chunk),
   239	                        "chunk_text_preview": chunk[:150],
   240	                        "total_chunks": len(text_chunks),
   241	                    }
   242	                )
   243	                return None
   244
   245	            # Strip WAV header (44 bytes) to get raw PCM data for stitching
   246	            pcm_data = audio_bytes[44:] if len(audio_bytes) > 44 else audio_bytes
   247	            audio_chunks.append(pcm_data)
   248
   249	        # 3. Stitch audio with crossfade
   250	        stitched_pcm = self.stitch_audio_chunks(
   251	            audio_chunks,
   252	            sample_rate=sample_rate,
   253	            crossfade_ms=self.CROSSFADE_MS,
   254	        )
   255
   256	        # 4. Add WAV header to stitched PCM data
   257	        wav_header = create_wav_header(
   258	            data_size=len(stitched_pcm),
   259	            sample_rate=sample_rate,
   260	            channels=1,
   261	            bits_per_sample=16
   262	        )
   263	        final_audio = wav_header + stitched_pcm
   264
   265	        # Calculate total audio metrics
   266	        total_audio_duration = len(stitched_pcm) / (sample_rate * 2)
   267
   268	        logger.info(
   269	            f"✅ Generated {len(text_chunks)} chunks → {total_audio_duration:.1f}s audio",
   270	            extra={
   271	                "total_chunks": len(text_chunks),
   272	                "audio_duration": total_audio_duration,
   273	                "audio_size_mb": len(final_audio) / (1024 * 1024),
   274	            }
   275	        )
   276
   277	        return final_audio
   278
   279	    def stitch_audio_chunks(
   280	        self,


Input:
*** Begin Patch
*** Update File: veena3modal/processing/long_text_processor.py
@@
-import asyncio
 import numpy as np
 import struct
+import statistics
+import time
 from typing import List, Optional, AsyncGenerator
 import logging
@@
     async def generate_with_chunking(
         self,
         text: str,
         speaker: str,
         temperature: float = 0.4,
@@
         max_tokens: int = 4096,
         repetition_penalty: float = 1.05,  # Prevent token repetition
         seed: Optional[int] = None,
         sample_rate: int = 16000,  # BiCodec uses 16kHz, not 24kHz
-    ) -> Optional[bytes]:
+        return_timing: bool = False,
+        request_id: Optional[str] = None,
+    ) -> Optional[bytes] | tuple[Optional[bytes], dict]:
@@
         Returns:
-            Stitched audio bytes (int16 PCM) or None if generation fails
+            Stitched audio bytes (int16 PCM) and optional timing metrics
         """
+        # Tracking variables for detailed timing aggregation
+        request_start = time.perf_counter()
+        emitted_chunk_metrics: List[dict] = []
+        llm_batch_wall_ms_values: List[float] = []
+        llm_batch_gpu_ms_values: List[float] = []
+        llm_decode_cpu_ms_values: List[float] = []
+        llm_parse_ms_values: List[float] = []
+        llm_batch_tokens: List[int] = []
+
         # Detect language mix for logging
         lang_info = self.chunker.detect_language_mix(text)
@@
-            # Generate audio for this chunk (returns WAV format with header)
-            audio_bytes = await self.pipeline.generate_speech_indic(
+            # Generate audio for this chunk (returns WAV format with header)
+            chunk_result = await self.pipeline.generate_speech_indic(
                 speaker=speaker,
                 text=chunk,
                 temperature=temperature,
                 top_k=top_k,  # Added for Spark TTS
                 top_p=top_p,
                 max_tokens=chunk_max_tokens,  # Use calculated limit, not global max_tokens
                 repetition_penalty=repetition_penalty,  # Prevent token repetition
                 seed=seed,  # Use same seed for consistent voice across chunks
+                request_id=request_id,
+                return_timing=return_timing,
             )
+
+            if return_timing:
+                if isinstance(chunk_result, tuple):
+                    audio_bytes, chunk_timing = chunk_result
+                    if isinstance(chunk_timing, dict):
+                        emitted_chunk_metrics.append(chunk_timing)
+                        llm_batch_wall_ms_values.append(int(chunk_timing.get("llm_wall_ms", 0)))
+                        llm_batch_gpu_ms_values.append(int(chunk_timing.get("llm_gpu_batch_ms_total", 0)))
+                        llm_decode_cpu_ms_values.append(int(chunk_timing.get("decode_cpu_ms", 0)))
+                        llm_parse_ms_values.append(int(chunk_timing.get("parse_ms", 0)))
+                        chunk_tokens = chunk_timing.get("token_batches", [])
+                        if isinstance(chunk_tokens, list):
+                            llm_batch_tokens.extend([int(x) for x in chunk_tokens if isinstance(x, int) or str(x).isdigit()])
+                else:
+                    audio_bytes = chunk_result
+            else:
+                audio_bytes = chunk_result

             if audio_bytes is None:
                 logger.error(
                     f"❌ Failed to generate audio for chunk {i+1}/{len(text_chunks)}",
@@
         logger.info(
             f"✅ Generated {len(text_chunks)} chunks → {total_audio_duration:.1f}s audio",
             extra={
@@
             }
         )

-        return final_audio
+        if return_timing:
+            request_ms = (time.perf_counter() - request_start) * 1000
+            chunked_metrics = {
+                "request_id": request_id,
+                "speaker": speaker,
+                "text_length": len(text),
+                "chunks_processed": len(text_chunks),
+                "audio_duration_seconds": total_audio_duration,
+                "audio_bytes": len(final_audio),
+                "text_chunked": True,
+                "token_total": sum(m.get("token_total", 0) for m in emitted_chunk_metrics),
+                "llm_batch_count": sum(int(m.get("batch_count", 0) or 0) for m in emitted_chunk_metrics),
+                "llm_batch_wall_ms_total": int(sum(llm_batch_wall_ms_values)),
+                "llm_batch_gpu_ms_total": int(sum(llm_batch_gpu_ms_values)),
+                "llm_decode_cpu_ms": int(sum(llm_decode_cpu_ms_values)),
+                "llm_parse_ms": int(sum(llm_parse_ms_values)),
+                "llm_batch_tokens": llm_batch_tokens,
+                "generation_ms": int(request_ms),
+            }
+            if llm_batch_wall_ms_values:
+                chunked_metrics["llm_batch_wall_ms_min"] = int(min(llm_batch_wall_ms_values))
+                chunked_metrics["llm_batch_wall_ms_max"] = int(max(llm_batch_wall_ms_values))
+                chunked_metrics["llm_batch_wall_ms_p50"] = int(statistics.median(llm_batch_wall_ms_values))
+            if llm_batch_gpu_ms_values:
+                chunked_metrics["llm_batch_gpu_ms_min"] = int(min(llm_batch_gpu_ms_values))
+                chunked_metrics["llm_batch_gpu_ms_max"] = int(max(llm_batch_gpu_ms_values))
+                chunked_metrics["llm_batch_gpu_ms_p50"] = int(statistics.median(llm_batch_gpu_ms_values))
+            if llm_batch_tokens:
+                chunked_metrics["tokens_per_batch_min"] = min(llm_batch_tokens)
+                chunked_metrics["tokens_per_batch_max"] = max(llm_batch_tokens)
+                chunked_metrics["tokens_per_batch_p50"] = int(statistics.median(llm_batch_tokens))
+            logger.info("tts_chunked_nonstream_complete", extra=chunked_metrics)
+            return final_audio, chunked_metrics
+
+        return final_audio
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/processing/long_text_processor.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


I’m now adding detailed timing/aggregation into streaming chunk methods (first chunk + continuation) because this is the main blind spot for GPU concurrency analysis under chunked streaming. I’ll also fix the malformed indentation block there.