Output:
Chunk ID: 14e4ea
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2963
Output:
"""
Spark TTS Generation Pipeline

End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
"""

import asyncio
import logging
import os
from statistics import mean, median
from typing import Optional, List, Dict, Any, Tuple

from vllm import SamplingParams
import torch

logger = logging.getLogger(__name__)

from veena3modal.core.constants import (
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_SEED,
    AUDIO_SAMPLE_RATE,
)

def _detailed_timing_enabled() -> bool:
    """
    Return True when benchmark logs are enabled.

    Controlled by environment variable:
      VEENA3_TIMING_LOGS=true|1
    """
    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}

class SparkTTSPipeline:
    """
    End-to-end TTS pipeline for Spark TTS with BiCodec.

    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        bicodec_decoder,
    ):
        """
        Initialize pipeline.

        Args:
            model: SparkTTS Model instance
            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
            bicodec_decoder: BiCodecDecoder instance
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.bicodec_decoder = bicodec_decoder

        print(f"🚀 SparkTTSPipeline initialized")

    async def generate_speech(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
        """
        Generate speech audio (non-streaming) using Spark TTS.

        NOTE: This method signature changed from description-based to speaker-based
        to align with Spark TTS architecture.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize (with optional [emotion] tags)
            temperature: Sampling temperature (default: 0.8 for Spark TTS)
            top_k: Top-k sampling (default: 50)
            top_p: Nucleus sampling (default: 1.0)
            max_tokens: Max BiCodec tokens to generate (default: 2048)
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Build prompt using Spark TTS format
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling for Spark TTS
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
        )

        # Use vLLM engine to generate
        import uuid
        request_id = f"req-{uuid.uuid4().hex[:12]}"

        emit_timing = return_timing or _detailed_timing_enabled()
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Collect all outputs and track generation timing
        final_output = None
        token_batches: List[int] = []
        batch_wall_ms: List[float] = []
        total_token_ids = 0
        first_token_ms: Optional[float] = None
        t_start = asyncio.get_event_loop().time()

        iterator = results_generator.__aiter__()
        while True:
            wall_start = asyncio.get_event_loop().time()
            start_event = None
            end_event = None
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                start_event = torch.cuda.Event(enable_timing=True)
                end_event = torch.cuda.Event(enable_timing=True)
                start_event.record()

            try:
                request_output = await iterator.__anext__()
            except StopAsyncIteration:
                break

            wall_end = asyncio.get_event_loop().time()
            batch_ms = (wall_end - wall_start) * 1000
            batch_wall_ms.append(batch_ms)

            if start_event is not None and end_event is not None:
                end_event.record()
                torch.cuda.synchronize()
                batch_gpu_ms = start_event.elapsed_time(end_event)
            else:
                batch_gpu_ms = None

            total_generated = len(request_output.outputs[0].token_ids)
            new_tokens = max(0, total_generated - total_token_ids)
            total_token_ids = total_generated
            token_batches.append(new_tokens)

            if first_token_ms is None:
                first_token_ms = (wall_end - t_start) * 1000

            if emit_timing:
                logger.info(
                    "tts_llm_batch",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "batch_index": len(token_batches),
                        "batch_tokens": new_tokens,
                        "llm_batch_wall_ms": round(batch_ms, 3),
                        "llm_batch_gpu_ms": round(batch_gpu_ms, 3) if batch_gpu_ms is not None else None,
                    },
                )
            final_output = request_output

        if final_output is None:
            if emit_timing:
                logger.warning(
                    "tts_llm_no_output",
                    extra={"speaker": speaker, "request_id": request_id},
                )
            print(f"❌ Generation failed: no output")
            if return_timing:
                return None, {"ttfb_ms": 0, "token_total": 0, "batch_count": 0}
            return None

        # Extract generated text (contains BiCodec tokens)
        parse_start = asyncio.get_event_loop().time()
        generated_text = final_output.outputs[0].text
        generated_ids = final_output.outputs[0].token_ids

        # Extract BiCodec tokens using regex
        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)

        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000

        if not semantic_ids or not global_ids:
            if emit_timing:
                logger.error(
                    "tts_bicodec_parse_failed",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "semantic_count": len(semantic_ids),
                        "global_count": len(global_ids),
                        "token_total": len(generated_ids),
                    },
                )
            logger.error(
                "❌ No BiCodec tokens found in output",
                extra={
                    "speaker": speaker,
                    "text_length": len(text),
                    "text_preview": text[:100],
                    "generated_tokens": len(generated_ids),
                    "semantic_tokens": len(semantic_ids),
                    "global_tokens": len(global_ids),
                    "generated_text_preview": generated_text[:500],
                }
            )
            if return_timing:
                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
            return None

        # Validate tokens
        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
            logger.error("❌ BiCodec token validation failed", extra={
                "speaker": speaker,
                "text_length": len(text),
                "semantic_tokens": len(semantic_ids),
                "global_tokens": len(global_ids),
            })
            if return_timing:
                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
            return None

        # Decode to audio
        decode_start = asyncio.get_event_loop().time()
        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000

        if audio_bytes is None:
            if emit_timing:
                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
            logger.error("❌ BiCodec decode failed", extra={
                "speaker": speaker,
                "text_length": len(text),
            })
            if return_timing:
                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
            return None

        total_ms = (asyncio.get_event_loop().time() - t_start) * 1000

        timing_metrics: Dict[str, Any] = {
            "request_id": request_id,
            "speaker": speaker,
            "text_length": len(text),
            "batch_count": len(token_batches),
            "token_total": len(generated_ids),
            "token_batches": token_batches,
            "ttfb_ms": int(first_token_ms or 0),
            "llm_wall_ms": int(sum(batch_wall_ms)),
            "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
            "decode_cpu_ms": int(decode_ms),
            "parse_ms": int(parse_ms),
            "total_ms": int(total_ms),
        }
        if token_batches:
            timing_metrics["tokens_per_batch_min"] = min(token_batches)
            timing_metrics["tokens_per_batch_max"] = max(token_batches)
            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))

        if emit_timing:
            logger.info("tts_llm_complete", extra=timing_metrics)

        from veena3modal.audio.utils import add_wav_header
        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)

        if return_timing:
            return wav_bytes, timing_metrics
        return wav_bytes

    async def generate_speech_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
    ):
        """
        Generate speech audio for Spark TTS (speaker-based).

        This method is kept for backward compatibility with existing API.
        It delegates to generate_speech() with the same implementation.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "Hello <laugh> this is fun!" (will be normalized)
                - "नमस्ते! [excited] आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max BiCodec tokens to generate
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '320,520p' /home/ubuntu/veenaModal/veena3modal/core/pipeline.py


Output:
Chunk ID: 4174e9
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 453
Output:
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Delegate to generate_speech() with the same implementation for Spark TTS
        return await self.generate_speech(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
            return_timing=return_timing,
        )

    def _extract_bicodec_tokens(self, generated_text: str) -> Tuple[List[int], List[int]]:
        """
        Extract BiCodec semantic and global tokens from generated text using regex.

        Spark TTS generates tokens in the format:
        - <|bicodec_semantic_{id}|>
        - <|bicodec_global_{id}|>

        Args:
            generated_text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
            Returns ([], []) if no tokens found
        """
        # Extract semantic tokens
        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", generated_text)
        semantic_ids = [int(t) for t in semantic_matches]

        # Extract global tokens
        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
        global_ids = [int(t) for t in global_matches]

        # Only log if there's an issue
        if not semantic_ids and not global_ids:
            logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")

        return semantic_ids, global_ids


Summary: auto


sed -n '1,260p' /home/ubuntu/veenaModal/veena3modal/tests/performance/test_benchmarks.py


Output:
Chunk ID: e57e46
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 2222
Output:
"""
Performance benchmarks for TTS service.

Tests marked with @pytest.mark.slow for CI filtering.
These tests require GPU and model to be loaded.

Run with: pytest veena3modal/tests/performance -v -m slow
"""

import pytest
import time
import asyncio
from typing import List, Tuple

# Skip all tests if model not available
def _can_run_benchmarks() -> Tuple[bool, str]:
    """Check if benchmarks can run."""
    try:
        import torch
        if not torch.cuda.is_available():
            return False, "CUDA not available"

        from veena3modal.services import tts_runtime
        if not tts_runtime.is_initialized():
            return False, "TTS runtime not initialized"

        return True, ""
    except Exception as e:
        return False, str(e)

CAN_RUN, SKIP_REASON = _can_run_benchmarks()

@pytest.fixture(scope="module")
def initialized_runtime():
    """Ensure runtime is initialized for benchmarks."""
    if not CAN_RUN:
        pytest.skip(f"Benchmarks skipped: {SKIP_REASON}")

    from veena3modal.services import tts_runtime

    if not tts_runtime.is_initialized():
        pytest.skip("TTS runtime not initialized")

    return tts_runtime.get_runtime()

@pytest.mark.slow
@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Benchmark prerequisites not met")
class TestTTFBBenchmarks:
    """
    Time-to-First-Byte benchmarks.

    Target: < 500ms warm, < 1200ms cold
    """

    @pytest.mark.asyncio
    async def test_ttfb_short_text_nonstreaming(self, initialized_runtime):
        """TTFB for short text non-streaming."""
        from veena3modal.services import tts_runtime

        text = "Hello, this is a test."

        start = time.time()
        audio_bytes, metrics = await tts_runtime.generate_speech(
            text=text,
            speaker="lipakshi",
        )
        ttfb = (time.time() - start) * 1000

        assert audio_bytes is not None
        assert ttfb < 2000, f"TTFB too high: {ttfb:.0f}ms"
        print(f"TTFB (short, non-streaming): {ttfb:.0f}ms")

    @pytest.mark.asyncio
    async def test_ttfb_streaming_first_chunk(self, initialized_runtime):
        """TTFB for streaming (time to first chunk)."""
        from veena3modal.services import tts_runtime

        text = "Hello, this is a streaming test."

        start = time.time()
        first_chunk_time = None

        async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
            text=text,
            speaker="lipakshi",
        ):
            if first_chunk_time is None:
                first_chunk_time = (time.time() - start) * 1000
            # Don't break - let it complete

        assert first_chunk_time is not None
        assert first_chunk_time < 1500, f"Streaming TTFB too high: {first_chunk_time:.0f}ms"
        print(f"TTFB (streaming, first chunk): {first_chunk_time:.0f}ms")

@pytest.mark.slow
@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Benchmark prerequisites not met")
class TestRTFBenchmarks:
    """
    Real-Time Factor benchmarks.

    RTF = generation_time / audio_duration
    Target: < 0.5 single request, < 0.8 under load
    """

    @pytest.mark.asyncio
    async def test_rtf_short_text(self, initialized_runtime):
        """RTF for short text."""
        from veena3modal.services import tts_runtime

        text = "Hello, this is a test sentence for measuring real-time factor."

        start = time.time()
        audio_bytes, metrics = await tts_runtime.generate_speech(
            text=text,
            speaker="lipakshi",
        )
        generation_time = time.time() - start

        audio_duration = metrics.get("audio_duration_seconds", 0)
        if audio_duration > 0:
            rtf = generation_time / audio_duration
            assert rtf < 1.5, f"RTF too high: {rtf:.2f}"
            print(f"RTF (short): {rtf:.2f} ({audio_duration:.1f}s audio in {generation_time:.1f}s)")

    @pytest.mark.asyncio
    async def test_rtf_long_text(self, initialized_runtime):
        """RTF for long text (with chunking)."""
        from veena3modal.services import tts_runtime

        # ~500 chars of text
        text = """
        This is a longer piece of text that will test the chunking capabilities
        of our TTS system. The text normalization should handle numbers like 12345
        and dates like January 15, 2024. It should also handle currency like $99.99
        and technical terms like API and HTTP. Let's see how well it performs with
        this amount of text that should trigger the chunking mechanism.
        """

        start = time.time()
        audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
            text=text,
            speaker="lipakshi",
        )
        generation_time = time.time() - start

        audio_duration = metrics.get("audio_duration_seconds", 0)
        if audio_duration > 0:
            rtf = generation_time / audio_duration
            assert rtf < 2.0, f"RTF too high for long text: {rtf:.2f}"
            print(f"RTF (long, chunked): {rtf:.2f} ({audio_duration:.1f}s audio in {generation_time:.1f}s)")

@pytest.mark.slow
@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Benchmark prerequisites not met")
class TestThroughputBenchmarks:
    """
    Throughput benchmarks.

    Tests concurrent request handling.
    """

    @pytest.mark.asyncio
    async def test_concurrent_requests(self, initialized_runtime):
        """Measure throughput with concurrent requests."""
        from veena3modal.services import tts_runtime

        texts = [
            "First test sentence.",
            "Second test sentence.",
            "Third test sentence.",
            "Fourth test sentence.",
            "Fifth test sentence.",
        ]

        async def generate(text):
            start = time.time()
            audio_bytes, metrics = await tts_runtime.generate_speech(
                text=text,
                speaker="lipakshi",
            )
            return time.time() - start, metrics.get("audio_duration_seconds", 0)

        start = time.time()
        results = await asyncio.gather(*[generate(t) for t in texts])
        total_time = time.time() - start

        gen_times = [r[0] for r in results]
        audio_durations = [r[1] for r in results]

        total_audio = sum(audio_durations)

        print(f"\nConcurrent throughput ({len(texts)} requests):")
        print(f"  Total wall time: {total_time:.1f}s")
        print(f"  Total audio generated: {total_audio:.1f}s")
        print(f"  Avg per-request time: {sum(gen_times)/len(gen_times):.1f}s")
        print(f"  Effective RTF: {total_time/total_audio:.2f}" if total_audio > 0 else "")

@pytest.mark.slow
@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Benchmark prerequisites not met")
class TestMemoryBenchmarks:
    """
    Memory usage benchmarks.
    """

    def test_gpu_memory_baseline(self, initialized_runtime):
        """Check GPU memory usage at rest."""
        import torch

        if not torch.cuda.is_available():
            pytest.skip("CUDA not available")

        allocated = torch.cuda.memory_allocated() / (1024 ** 3)  # GB
        reserved = torch.cuda.memory_reserved() / (1024 ** 3)  # GB

        print(f"\nGPU Memory (baseline):")
        print(f"  Allocated: {allocated:.2f} GB")
        print(f"  Reserved: {reserved:.2f} GB")

        # Should be reasonable for inference
        assert reserved < 40, f"GPU memory too high: {reserved:.2f} GB"

    @pytest.mark.asyncio
    async def test_gpu_memory_after_generation(self, initialized_runtime):
        """Check GPU memory after generation."""
        import torch
        from veena3modal.services import tts_runtime

        if not torch.cuda.is_available():
            pytest.skip("CUDA not available")

        # Generate some audio
        for i in range(3):
            await tts_runtime.generate_speech(
                text=f"Test sentence number {i+1}.",
                speaker="lipakshi",
            )

        allocated = torch.cuda.memory_allocated() / (1024 ** 3)
        reserved = torch.cuda.memory_reserved() / (1024 ** 3)

        print(f"\nGPU Memory (after 3 generations):")
        print(f"  Allocated: {allocated:.2f} GB")
        print(f"  Reserved: {reserved:.2f} GB")

        # Memory should not leak significantly
        assert reserved < 50, f"GPU memory too high after generation: {reserved:.2f} GB"

@pytest.mark.slow
@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Benchmark prerequisites not met")
class TestCacheBenchmarks:
    """
    Cache performance benchmarks.
    """


sed -n '260,520p' /home/ubuntu/veenaModal/veena3modal/tests/performance/test_benchmarks.py


Output:
Chunk ID: f5f3f8
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 392
Output:
    """

    def test_rate_limiter_check_performance(self):
        """Rate limiter check should be < 1ms."""
        from veena3modal.api.rate_limiter import RateLimiter

        limiter = RateLimiter(requests_per_minute=1000)

        times = []
        for i in range(100):
            start = time.time()
            limiter.check(f"key_{i % 10}")
            times.append((time.time() - start) * 1000)

        avg_time = sum(times) / len(times)
        max_time = max(times)

        print(f"\nRate limiter check performance:")
        print(f"  Avg: {avg_time:.3f}ms")
        print(f"  Max: {max_time:.3f}ms")

        assert avg_time < 1, f"Rate limiter too slow: {avg_time:.3f}ms avg"

    def test_api_key_cache_performance(self):
        """API key cache lookup should be < 1ms."""
        from veena3modal.api.auth import ApiKeyCache

        cache = ApiKeyCache()

        # Populate cache
        for i in range(100):
            cache.set(f"key_{i}", {"user_id": f"user-{i}", "credits": 100})

        times = []
        for i in range(100):
            start = time.time()
            cache.get(f"key_{i}")
            times.append((time.time() - start) * 1000)

        avg_time = sum(times) / len(times)
        max_time = max(times)

        print(f"\nAPI key cache lookup performance:")
        print(f"  Avg: {avg_time:.3f}ms")
        print(f"  Max: {max_time:.3f}ms")

        assert avg_time < 1, f"Cache lookup too slow: {avg_time:.3f}ms avg"


sed -n '1,320p' /home/ubuntu/veenaModal/veena3modal/local_server.py


Output:
Chunk ID: c0ad38
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 2915
Output:
"""
Local entry point for Veena3 TTS - runs the same FastAPI app without Modal.

Replicates what Modal's @modal.enter + @modal.asgi_app does:
1. Sets up PYTHONPATH for vendored deps (sparktts, AP-BWE)
2. Downloads model weights from HuggingFace if not present
3. Initializes TTS runtime (vLLM engine, BiCodec decoder, pipelines)
4. Serves the FastAPI app via uvicorn

Usage:
    # Basic (auto-downloads model, uses defaults)
    python -m veena3modal.local_server

    # Custom model path (skip download)
    python -m veena3modal.local_server --model-path /path/to/spark_tts_4speaker

    # Custom GPU memory + port
    python -m veena3modal.local_server --gpu-memory 0.5 --port 8080

    # With super-resolution (48kHz output)
    python -m veena3modal.local_server --enable-sr --sr-path /path/to/ap_bwe/16kto48k

    # CPU-only (no GPU, for testing only - very slow)
    python -m veena3modal.local_server --device cpu

Environment Variables (override defaults):
    MODEL_PATH              - Path to Spark TTS model directory
    HF_TOKEN                - HuggingFace token for private models
    AP_BWE_CHECKPOINT_DIR   - Path to super-resolution checkpoints
    AUTH_BYPASS_MODE        - "true" to disable auth (default for local)
    GPU_MEMORY_UTILIZATION  - vLLM GPU memory fraction (default: 0.25)
"""

from __future__ import annotations

import argparse
import logging
import os
import sys
import time

# === Path Setup (MUST happen before any veena3modal imports) ===
# Vendored deps: sparktts and AP-BWE live in external/ relative to repo root
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
EXTERNAL_DIR = os.path.join(REPO_ROOT, "external")

# sparktts package path: external/sparktts (contains sparktts/ subpackage)
SPARKTTS_PATH = os.path.join(EXTERNAL_DIR, "sparktts")
# AP-BWE path: external/AP-BWE
AP_BWE_PATH = os.path.join(EXTERNAL_DIR, "AP-BWE")

for path in [SPARKTTS_PATH, AP_BWE_PATH, REPO_ROOT]:
    if path not in sys.path:
        sys.path.insert(0, path)

# Also set PYTHONPATH for subprocesses (e.g., ffmpeg wrappers)
existing_pythonpath = os.environ.get("PYTHONPATH", "")
new_paths = f"{SPARKTTS_PATH}:{AP_BWE_PATH}:{REPO_ROOT}"
if existing_pythonpath:
    os.environ["PYTHONPATH"] = f"{new_paths}:{existing_pythonpath}"
else:
    os.environ["PYTHONPATH"] = new_paths

logger = logging.getLogger("veena3.local")

# Default local model directory (relative to repo)
DEFAULT_LOCAL_MODEL_DIR = os.path.join(REPO_ROOT, "models", "spark_tts_4speaker")

# Model sources (tried in order):
# 1. Local path (--model-path or MODEL_PATH env)
# 2. HuggingFace download (private repo: BayAreaBoys/spark_tts_4speaker)
# 3. Base Spark TTS model (public: SparkAudio/Spark-TTS-0.5B)
PRIVATE_HF_REPO = "BayAreaBoys/spark_tts_4speaker"
PUBLIC_FALLBACK_REPO = "SparkAudio/Spark-TTS-0.5B"

def download_model(
    hf_repo: str = PRIVATE_HF_REPO,
    local_dir: str = DEFAULT_LOCAL_MODEL_DIR,
    hf_token: str | None = None,
) -> str:
    """
    Download Spark TTS model from HuggingFace if not already present.

    Strategy:
    1. Try private repo (BayAreaBoys/spark_tts_4speaker) with HF token
    2. Fall back to public base model (SparkAudio/Spark-TTS-0.5B)
    3. Provide instructions for Modal volume download if both fail

    Returns the resolved local path.
    """
    # Check for any valid marker file (config.json at root, config.yaml, or LLM/config.json)
    marker_files = ["config.json", "config.yaml", "LLM/config.json"]
    if any(os.path.exists(os.path.join(local_dir, m)) for m in marker_files):
        logger.info(f"Model already present at {local_dir}")
        return local_dir

    try:
        from huggingface_hub import snapshot_download
    except ImportError:
        logger.error(
            "huggingface_hub not installed. Install with: pip install huggingface-hub\n"
            "Or download model manually and pass --model-path"
        )
        sys.exit(1)

    os.makedirs(local_dir, exist_ok=True)

    # Try private repo first
    logger.info(f"Attempting download from {hf_repo}...")
    try:
        downloaded_path = snapshot_download(
            repo_id=hf_repo,
            local_dir=local_dir,
            token=hf_token,
            ignore_patterns=["*.bin", "training_args.bin", "optimizer.pt"],
        )
        logger.info(f"Model downloaded to {downloaded_path}")
        return downloaded_path
    except Exception as e:
        logger.warning(f"Private repo download failed: {e}")

    # Try public fallback
    if hf_repo != PUBLIC_FALLBACK_REPO:
        logger.info(f"Trying public base model: {PUBLIC_FALLBACK_REPO}...")
        try:
            downloaded_path = snapshot_download(
                repo_id=PUBLIC_FALLBACK_REPO,
                local_dir=local_dir,
                token=hf_token,
            )
            logger.info(f"Base model downloaded to {downloaded_path}")
            logger.warning(
                "Using public base SparkTTS model (no custom speakers). "
                "For full speaker support, download your fine-tuned model."
            )
            return downloaded_path
        except Exception as e2:
            logger.warning(f"Public model download also failed: {e2}")

    # All downloads failed - provide manual instructions
    logger.error(
        "\nModel download failed. Options:\n"
        "  1. Download from Modal volume:\n"
        "     modal volume get veena3-models spark_tts_4speaker models/spark_tts_4speaker\n\n"
        "  2. Download from HuggingFace (if you have access):\n"
        f"     huggingface-cli download {PRIVATE_HF_REPO} --local-dir {local_dir}\n\n"
        "  3. Use base SparkTTS model:\n"
        f"     huggingface-cli download {PUBLIC_FALLBACK_REPO} --local-dir {local_dir}\n\n"
        "  4. Specify a custom path:\n"
        "     python -m veena3modal.local_server --model-path /path/to/model"
    )
    sys.exit(1)

def resolve_model_paths(model_path: str) -> tuple[str, str]:
    """
    Resolve LLM and BiCodec paths from a Spark TTS model directory.

    Spark TTS model structure (from SparkAudio/Spark-TTS-0.5B):
      model_path/
        LLM/          <- Language model (vLLM loads this)
        BiCodec/      <- Audio tokenizer/decoder
        config.yaml   <- Top-level config

    Fine-tuned models may have LLM files at root (flat structure).
    This function auto-detects both layouts.

    Returns:
        (llm_path, bicodec_path) resolved paths
    """
    # Check for subdirectory layout (SparkAudio/Spark-TTS-0.5B style)
    llm_subdir = os.path.join(model_path, "LLM")
    bicodec_subdir = os.path.join(model_path, "BiCodec")

    if os.path.isdir(llm_subdir) and os.path.exists(os.path.join(llm_subdir, "config.json")):
        llm_path = llm_subdir
    else:
        # Flat layout: LLM files at root
        llm_path = model_path

    # BiCodec path is always the root (it contains BiCodec/ subdir)
    bicodec_path = model_path

    logger.info(f"Resolved model paths:")
    logger.info(f"  LLM (vLLM):    {llm_path}")
    logger.info(f"  BiCodec:       {bicodec_path}")

    return llm_path, bicodec_path

def initialize_local_runtime(
    model_path: str,
    device: str = "cuda",
    gpu_memory_utilization: float = 0.85,
    enable_sr: bool = False,
    sr_checkpoint_dir: str | None = None,
    hf_token: str | None = None,
    num_engines: int = 1,
):
    """
    Initialize TTS runtime for local serving.

    This replaces Modal's @modal.enter lifecycle hook.
    Loads: vLLM engine, BiCodec decoder, prompt builder, pipelines.
    """
    from veena3modal.services.tts_runtime import initialize_runtime

    # Resolve LLM vs BiCodec paths from model directory
    llm_path, bicodec_path = resolve_model_paths(model_path)

    start = time.time()
    logger.info("=" * 60)
    logger.info("Initializing Veena3 TTS Runtime (local)")
    logger.info(f"  Model:       {model_path}")
    logger.info(f"  LLM path:    {llm_path}")
    logger.info(f"  BiCodec:     {bicodec_path}")
    logger.info(f"  Device:      {device}")
    logger.info(f"  GPU Memory:  {gpu_memory_utilization:.0%} (total)")
    logger.info(f"  Engines:     {num_engines}")
    logger.info(f"  SR:          {'enabled' if enable_sr else 'disabled'}")
    logger.info("=" * 60)

    runtime = initialize_runtime(
        model_path=llm_path,
        bicodec_path=bicodec_path,
        sr_checkpoint_dir=sr_checkpoint_dir if enable_sr else None,
        device=device,
        hf_token=hf_token,
        gpu_memory_utilization=gpu_memory_utilization,
        enable_sr=enable_sr,
        num_engines=num_engines,
    )

    elapsed = time.time() - start
    logger.info(f"Runtime initialized in {elapsed:.1f}s")
    logger.info(f"  Model version: {runtime.model_version}")
    logger.info(f"  Streaming: {'available' if runtime.streaming_pipeline else 'unavailable'}")
    logger.info(f"  Super-resolution: {'loaded' if runtime.sr_service and runtime.sr_service.is_loaded else 'disabled'}")

    return runtime

def create_local_app():
    """
    Create the FastAPI app for local serving.

    Same app factory as Modal deployment, but called from local context.
    """
    from veena3modal.api.fastapi_app import create_app
    return create_app()

def parse_args():
    """Parse CLI arguments for local server configuration."""
    parser = argparse.ArgumentParser(
        description="Veena3 TTS Local Server - GPU-accelerated text-to-speech",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python -m veena3modal.local_server                          # Auto-download model, serve on :8000
  python -m veena3modal.local_server --port 8080              # Custom port
  python -m veena3modal.local_server --model-path ./my_model  # Use local model
  python -m veena3modal.local_server --gpu-memory 0.5         # Limit GPU memory
  python -m veena3modal.local_server --workers 2              # Multiple workers (careful: each loads model)
        """,
    )

    # Model configuration
    model_group = parser.add_argument_group("Model")
    model_group.add_argument(
        "--model-path",
        type=str,
        default=os.environ.get("MODEL_PATH", DEFAULT_LOCAL_MODEL_DIR),
        help=f"Path to Spark TTS model (default: {DEFAULT_LOCAL_MODEL_DIR})",
    )
    model_group.add_argument(
        "--hf-repo",
        type=str,
        default=PRIVATE_HF_REPO,
        help=f"HuggingFace repo to download from (default: {PRIVATE_HF_REPO})",
    )
    model_group.add_argument(
        "--hf-token",
        type=str,
        default=os.environ.get("HF_TOKEN"),
        help="HuggingFace token (default: from HF_TOKEN env var)",
    )
    model_group.add_argument(
        "--skip-download",
        action="store_true",
        help="Don't auto-download model (fail if not found)",
    )

    # GPU configuration
    gpu_group = parser.add_argument_group("GPU")
    gpu_group.add_argument(
        "--device",
        type=str,
        default="cuda",
        choices=["cuda", "cpu"],
        help="Device for inference (default: cuda)",
    )
    gpu_group.add_argument(
        "--gpu-memory",
        type=float,
        default=float(os.environ.get("GPU_MEMORY_UTILIZATION", "0.25")),
        help="vLLM GPU memory utilization fraction (default: 0.25)",
    )
    gpu_group.add_argument(
        "--num-engines",
        type=int,
        default=int(os.environ.get("NUM_ENGINES", "1")),
        help="Number of vLLM engine instances (default: 1, use 2-3 for multi-engine mode)",
    )

    # Super-resolution
    sr_group = parser.add_argument_group("Super-Resolution")
    sr_group.add_argument(
        "--enable-sr",


sed -n '320,760p' /home/ubuntu/veenaModal/veena3modal/local_server.py


Output:
Chunk ID: 2bb954
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 1129
Output:
        "--enable-sr",
        action="store_true",
        default=False,
        help="Enable AP-BWE super-resolution (16kHz -> 48kHz)",
    )
    sr_group.add_argument(
        "--sr-path",
        type=str,
        default=os.environ.get("AP_BWE_CHECKPOINT_DIR"),
        help="Path to AP-BWE checkpoint directory",
    )

    # Server configuration
    server_group = parser.add_argument_group("Server")
    server_group.add_argument(
        "--host",
        type=str,
        default="0.0.0.0",
        help="Bind address (default: 0.0.0.0)",
    )
    server_group.add_argument(
        "--port",
        type=int,
        default=int(os.environ.get("PORT", "8000")),
        help="Port to serve on (default: 8000)",
    )
    server_group.add_argument(
        "--workers",
        type=int,
        default=1,
        help="Number of uvicorn workers (default: 1, each loads full model)",
    )
    server_group.add_argument(
        "--reload",
        action="store_true",
        default=False,
        help="Enable auto-reload for development (incompatible with --workers > 1)",
    )

    # Auth
    auth_group = parser.add_argument_group("Auth")
    auth_group.add_argument(
        "--auth",
        action="store_true",
        default=False,
        help="Enable API key authentication (default: disabled for local)",
    )

    # Logging
    log_group = parser.add_argument_group("Logging")
    log_group.add_argument(
        "--log-level",
        type=str,
        default="INFO",
        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
        help="Log level (default: INFO)",
    )

    return parser.parse_args()

def main():
    """
    Main entry point for local TTS server.

    Lifecycle:
    1. Parse args + configure logging
    2. Download model if needed
    3. Initialize TTS runtime (vLLM engine, decoders, pipelines)
    4. Create FastAPI app
    5. Run uvicorn
    """
    args = parse_args()

    # Configure logging
    logging.basicConfig(
        level=getattr(logging, args.log_level),
        format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )

    # Auth bypass for local development (unless --auth flag is set)
    if not args.auth:
        os.environ["AUTH_BYPASS_MODE"] = "true"
        logger.info("Auth bypass enabled (local mode). Use --auth to require API keys.")
    else:
        os.environ.setdefault("AUTH_BYPASS_MODE", "false")

    # Resolve model path
    # Spark TTS root has config.yaml; LLM subdir has config.json; fine-tuned may have config.json at root
    model_path = args.model_path
    model_exists = any(
        os.path.exists(os.path.join(model_path, marker))
        for marker in ("config.json", "config.yaml", "LLM/config.json")
    )
    if not model_exists:
        if args.skip_download:
            logger.error(f"Model not found at {model_path} and --skip-download is set")
            sys.exit(1)
        model_path = download_model(
            hf_repo=args.hf_repo,
            local_dir=model_path,
            hf_token=args.hf_token,
        )

    # Initialize TTS runtime (this is the expensive step - loads model onto GPU)
    initialize_local_runtime(
        model_path=model_path,
        device=args.device,
        gpu_memory_utilization=args.gpu_memory,
        enable_sr=args.enable_sr,
        sr_checkpoint_dir=args.sr_path,
        hf_token=args.hf_token,
        num_engines=args.num_engines,
    )

    # Run uvicorn
    # NOTE: We use the factory string so uvicorn can find the app.
    # With --reload, uvicorn reimports, but the runtime singleton persists in memory.
    import uvicorn

    logger.info("")
    logger.info("=" * 60)
    logger.info(f"Veena3 TTS Local Server starting")
    logger.info(f"  URL: http://{args.host}:{args.port}")
    logger.info(f"  Health: http://{args.host}:{args.port}/v1/tts/health")
    logger.info(f"  Generate: POST http://{args.host}:{args.port}/v1/tts/generate")
    logger.info(f"  Metrics: http://{args.host}:{args.port}/v1/tts/metrics")
    logger.info(f"  Docs: http://{args.host}:{args.port}/docs")
    logger.info("=" * 60)

    uvicorn.run(
        "veena3modal.local_server:create_local_app",
        factory=True,
        host=args.host,
        port=args.port,
        workers=args.workers,
        reload=args.reload,
        log_level=args.log_level.lower(),
        # Performance: keep-alive for connection reuse under load
        timeout_keep_alive=30,
    )

if __name__ == "__main__":
    main()


Output:
Chunk ID: 57fa13
Wall time: 0.0509 seconds
Process exited with code 2
Original token count: 46
Output:
rg: regex parse error:
    (?:generate_speech_stream_indic(_first_chunk|_continuation|\(|_stream\_chunked_text|streaming_pipeline.generate_speech_stream)
    ^
error: unclosed group


I found a regex typo in a quick search command; I’m doing a clean call now to map every existing callsite before editing signatures so we don’t introduce breaking calls.


rg -n "generate_speech_stream_indic|_stream_chunked_text|generate_speech_stream" /home/ubuntu/veenaModal/veena3modal -g'*.py'


Output:
Chunk ID: 893af1
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 715
Output:
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:705:async def generate_speech_streaming(
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:790:            async for audio_chunk, chunk_metrics in _stream_chunked_text(
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:827:                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:840:                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:878:async def _stream_chunked_text(
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:914:            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:942:                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:963:                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
/home/ubuntu/veenaModal/veena3modal/processing/long_text_processor.py:131:        This method is used by _stream_chunked_text in tts_runtime.
/home/ubuntu/veenaModal/veena3modal/processing/long_text_processor.py:380:            async for audio_bytes in self.streaming_pipeline.generate_speech_stream_indic(
/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py:550:                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
/home/ubuntu/veenaModal/veena3modal/api/websocket_handler.py:342:            async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
/home/ubuntu/veenaModal/veena3modal/core/dual_engine.py:126:# 2. In streaming_pipeline.py, modify generate_speech_stream_indic():
/home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:94:    async def generate_speech_stream(
/home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:222:    async def generate_speech_stream_indic(
/home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:503:    async def generate_speech_stream_indic_first_chunk(
/home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:519:        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.
/home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:523:            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
/home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:699:    async def generate_speech_stream_indic_continuation(
/home/ubuntu/veenaModal/veena3modal/tests/performance/test_benchmarks.py:87:        async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(


rg --files /home/ubuntu/veenaModal/scripts /home/ubuntu/veenaModal | head -n 80


Output:
Chunk ID: b34612
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1175
Output:
/home/ubuntu/veenaModal/requirements.txt
/home/ubuntu/veenaModal/.env.example
/home/ubuntu/veenaModal/modal_docs/# Managing deployments.md
/home/ubuntu/veenaModal/modal_docs/# Reserving CPU and memory.md
/home/ubuntu/veenaModal/modal_docs/# Job processing.md
/home/ubuntu/veenaModal/modal_docs/# Secrets.md
/home/ubuntu/veenaModal/modal_docs/# Using CUDA on Modal.md
/home/ubuntu/veenaModal/modal_docs/# Introduction.md
/home/ubuntu/veenaModal/scripts/detailed_word_analysis.py
/home/ubuntu/veenaModal/scripts/measure_ttfb_detailed_v2.py
/home/ubuntu/veenaModal/scripts/analyze_missing_words.py
/home/ubuntu/veenaModal/scripts/create_visual_map.py
/home/ubuntu/veenaModal/scripts/pattern_analysis.py
/home/ubuntu/veenaModal/scripts/validate_true_streaming.py
/home/ubuntu/veenaModal/scripts/measure_client_ttfb.py
/home/ubuntu/veenaModal/scripts/quick_asr_check.py
/home/ubuntu/veenaModal/scripts/stress_test_production.py
/home/ubuntu/veenaModal/scripts/asr_with_openai_final.py
/home/ubuntu/veenaModal/scripts/validation/final_validation.py
/home/ubuntu/veenaModal/scripts/elevenlabs_verify_fix.py
/home/ubuntu/veenaModal/scripts/validate_indic_normalization.py
/home/ubuntu/veenaModal/scripts/validate_text_normalization.py
/home/ubuntu/veenaModal/scripts/test_chunking_no_emotions.py
/home/ubuntu/veenaModal/scripts/test_true_streaming.py
/home/ubuntu/veenaModal/scripts/elevenlabs_asr_analysis.py
/home/ubuntu/veenaModal/scripts/validate_modal_endpoint.py
/home/ubuntu/veenaModal/scripts/asr_validation_openai.py
/home/ubuntu/veenaModal/scripts/wer_test_with_emotions.py
/home/ubuntu/veenaModal/scripts/test_fix.py
/home/ubuntu/veenaModal/scripts/profile_pipeline.py
/home/ubuntu/veenaModal/scripts/final_asr_verification.py
/home/ubuntu/veenaModal/scripts/test_all_features.py
/home/ubuntu/veenaModal/scripts/setup_local.sh
/home/ubuntu/veenaModal/scripts/comprehensive_validation.py
/home/ubuntu/veenaModal/scripts/stress_test_local.py
/home/ubuntu/veenaModal/scripts/test_correct_format.py
/home/ubuntu/veenaModal/scripts/test_original_bug.py
/home/ubuntu/veenaModal/scripts/final_chunking_test.sh
/home/ubuntu/veenaModal/scripts/validate_chunking_asr.py
/home/ubuntu/veenaModal/scripts/measure_ttfb_detailed.py
/home/ubuntu/veenaModal/modal_docs/# Using existing images.md
/home/ubuntu/veenaModal/modal_docs/# Hello world wide web!.md
/home/ubuntu/veenaModal/modal_docs/# Failures and retries.md
/home/ubuntu/veenaModal/modal_docs/# Scheduling remote cron jobs.md
/home/ubuntu/veenaModal/modal_docs/# Scaling out.md
/home/ubuntu/veenaModal/modal_docs/# Storing model weights on Modal.md
/home/ubuntu/veenaModal/modal_docs/# Images.md
/home/ubuntu/veenaModal/modal_docs/# Cold start performance.md
/home/ubuntu/veenaModal/modal_docs/# Input concurrency.md
/home/ubuntu/veenaModal/modal_docs/# Continuous deployment.md
/home/ubuntu/veenaModal/modal_docs/# Dicts.md
/home/ubuntu/veenaModal/modal_docs/# Multi-node clusters (beta).md
/home/ubuntu/veenaModal/modal_docs/# Batch Processing.md
/home/ubuntu/veenaModal/modal_docs/# Slack notifications (beta).md
/home/ubuntu/veenaModal/modal_docs/# Volumes.md
/home/ubuntu/veenaModal/modal_docs/# Apps, Functions, and entrypoints.md
/home/ubuntu/veenaModal/modal_docs/# Environment variables.md
/home/ubuntu/veenaModal/modal_docs/# Run a FastRTC app on Modal.md
/home/ubuntu/veenaModal/modal_docs/# GPU acceleration.md
/home/ubuntu/veenaModal/modal_docs/# Fast pull from registry.md
/home/ubuntu/veenaModal/modal_docs/# Real-time object detection with WebRTC.md
/home/ubuntu/veenaModal/modal_docs/# Memory Snapshot.md
/home/ubuntu/veenaModal/modal_docs/# GPU Health.md
/home/ubuntu/veenaModal/modal_docs/# Queues.md
/home/ubuntu/veenaModal/modal_docs/# Asynchronous API usage.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.Volume.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.config.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.enter.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.Function.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.web_server.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.forward.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.Error.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.Image.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.exception.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.gpu.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.file_io.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.Tunnel.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.parameter.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.Dict.md
/home/ubuntu/veenaModal/modal_docs/Reference/# modal.current_function_call_id.md


sed -n '1,320p' /home/ubuntu/veenaModal/scripts/stress_test_local.py


Output:
Chunk ID: a54154
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2590
Output:
#!/usr/bin/env python3
"""
Local Stress Test for Veena3 TTS Server.

Progressive load ramp with GPU monitoring, latency percentiles,
error categorization, and throughput measurement.

Unlike the Modal load tests, this runs locally with:
- No network latency (localhost)
- Direct GPU monitoring (nvidia-smi + torch.cuda)
- Higher concurrency ceiling (single-machine, no rate limits)
- Real streaming TTFB measurement (byte-level timing)

Usage:
    # Full suite (ramps 1 → 5 → 10 → 20 → 50 → 100 concurrent)
    python scripts/stress_test_local.py

    # Quick smoke test (1 → 5 → 10 only)
    python scripts/stress_test_local.py --quick

    # Custom ramp
    python scripts/stress_test_local.py --levels 1,5,10,20

    # Streaming only
    python scripts/stress_test_local.py --stream-only

    # Non-streaming only
    python scripts/stress_test_local.py --no-stream

    # Custom endpoint
    python scripts/stress_test_local.py --url http://localhost:8080
"""

import argparse
import asyncio
import json
import os
import statistics
import subprocess
import sys
import time
import threading
from dataclasses import dataclass, field
from typing import List, Optional, Dict, Any, Tuple

import httpx

# === Configuration ===

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

# Test texts at different lengths to stress different parts of the pipeline
TEST_TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This sentence tests basic text normalization and generation quality "
        "with moderate length input that should complete in reasonable time."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast long shadows "
        "over crowded streets, there lived an old bookkeeper named Margaret. She had spent "
        "forty years cataloging the stories of others, yet never found time to write her own. "
        "One rainy Tuesday afternoon, she discovered a leather-bound journal tucked behind "
        "a shelf of dusty encyclopedias. Its pages were blank, waiting. She picked up her pen "
        "and began to write. The words flowed like water from a broken dam."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

# === GPU Monitoring ===

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    """Background thread that polls nvidia-smi for GPU stats."""

    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=5)

    def _poll_loop(self):
        while self._running:
            try:
                result = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True, text=True, timeout=5,
                )
                if result.returncode == 0:
                    parts = result.stdout.strip().split(", ")
                    if len(parts) >= 4:
                        self.snapshots.append(GPUSnapshot(
                            timestamp=time.time(),
                            memory_used_mb=float(parts[0]),
                            memory_total_mb=float(parts[1]),
                            gpu_utilization_pct=float(parts[2]),
                            temperature_c=float(parts[3]),
                        ))
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {"error": "no GPU data collected"}

        mem_used = [s.memory_used_mb for s in self.snapshots]
        gpu_util = [s.gpu_utilization_pct for s in self.snapshots]
        temps = [s.temperature_c for s in self.snapshots]

        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem_used),
                "max": max(mem_used),
                "avg": statistics.mean(mem_used),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(gpu_util),
                "max": max(gpu_util),
                "avg": statistics.mean(gpu_util),
            },
            "temperature_c": {
                "min": min(temps),
                "max": max(temps),
                "avg": statistics.mean(temps),
            },
        }

# === Request Result ===

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float  # Total wall-clock time
    ttfb_ms: float = 0.0  # Time to first byte (from HTTP response start)
    server_ttfb_ms: float = 0.0  # Server-reported TTFB (X-TTFB-ms header)
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    error: Optional[str] = None
    speaker: str = ""
    text_length: int = 0
    stream: bool = False

# === Load Level Result ===

@dataclass
class LevelResult:
    level_name: str
    concurrency: int
    total_requests: int
    mode: str  # "non-streaming" or "streaming"
    text_category: str  # "short", "medium", "long", "mixed"
    results: List[RequestResult] = field(default_factory=list)
    wall_time_seconds: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_seconds if self.wall_time_seconds > 0 else 0

    def _percentile(self, values: List[float], pct: float) -> float:
        if not values:
            return 0.0
        s = sorted(values)
        idx = int(len(s) * pct / 100)
        return s[min(idx, len(s) - 1)]

    @property
    def latency_stats(self) -> Dict[str, float]:
        lats = [r.latency_ms for r in self.successes]
        if not lats:
            return {}
        return {
            "min": min(lats),
            "avg": statistics.mean(lats),
            "p50": self._percentile(lats, 50),
            "p95": self._percentile(lats, 95),
            "p99": self._percentile(lats, 99),
            "max": max(lats),
        }

    @property
    def ttfb_stats(self) -> Dict[str, float]:
        ttfbs = [r.ttfb_ms for r in self.successes if r.ttfb_ms > 0]
        if not ttfbs:
            return {}
        return {
            "min": min(ttfbs),
            "avg": statistics.mean(ttfbs),
            "p50": self._percentile(ttfbs, 50),
            "p95": self._percentile(ttfbs, 95),
            "max": max(ttfbs),
        }

    @property
    def audio_duration_total(self) -> float:
        return sum(r.audio_seconds for r in self.successes)

    @property
    def effective_rtf(self) -> float:
        """Wall time / total audio produced. Lower = better."""
        total_audio = self.audio_duration_total
        return self.wall_time_seconds / total_audio if total_audio > 0 else 0

    def error_summary(self) -> Dict[str, int]:
        errors: Dict[str, int] = {}
        for r in self.failures:
            key = r.error or f"HTTP {r.status_code}"
            errors[key[:80]] = errors.get(key[:80], 0) + 1
        return errors

# === HTTP Client ===

async def make_request(
    client: httpx.AsyncClient,
    base_url: str,
    text: str,
    speaker: str,
    stream: bool = False,
    timeout: float = 120.0,
) -> RequestResult:
    """Make a single TTS request with timing."""
    url = f"{base_url}{GENERATE_PATH}"
    payload = {
        "text": text,
        "speaker": speaker,
        "stream": stream,
        "format": "wav",
    }

    start = time.time()
    ttfb_time = 0.0

    try:
        if stream:
            # Streaming: measure true TTFB (time to first byte of audio)
            async with client.stream(
                "POST", url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            ) as response:
                first_byte = True
                total_bytes = 0
                async for chunk in response.aiter_bytes():
                    if first_byte:
                        ttfb_time = (time.time() - start) * 1000
                        first_byte = False
                    total_bytes += len(chunk)

                latency = (time.time() - start) * 1000

                if response.status_code == 200:
                    # Estimate audio duration from bytes (16kHz, 16-bit, mono, minus WAV header)
                    pcm_bytes = max(0, total_bytes - 44)
                    audio_seconds = pcm_bytes / (16000 * 2)

                    return RequestResult(
                        success=True,
                        status_code=200,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        audio_bytes=total_bytes,
                        audio_seconds=audio_seconds,
                        speaker=speaker,
                        text_length=len(text),
                        stream=True,
                    )
                else:
                    return RequestResult(
                        success=False,
                        status_code=response.status_code,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        error=f"HTTP {response.status_code}",
                        speaker=speaker,


sed -n '1,320p' /home/ubuntu/veenaModal/veena3modal/core/bicodec_decoder.py


Output:
Chunk ID: 8dc57c
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 3103
Output:
"""
Spark TTS BiCodec Decoder

Decodes BiCodec audio tokens (semantic + global) to audio waveforms.
Replaces SNAC decoder for Spark TTS model.
"""

import torch
import numpy as np
import sys
from pathlib import Path
from typing import List, Optional, Tuple

# Ensure `sparktts` (vendored under external/) is importable in local/dev runs.
# In Modal, PYTHONPATH already includes `/root/external/sparktts`.
try:
    from sparktts.models.audio_tokenizer import BiCodecTokenizer
except ImportError:  # pragma: no cover - environment-dependent
    _this_file = Path(__file__).resolve()
    _sparktts_path = None
    for _p in [_this_file.parent] + list(_this_file.parents):
        _candidate = _p / "external" / "sparktts"
        if _candidate.is_dir():
            _sparktts_path = str(_candidate)
            break
    if _sparktts_path and _sparktts_path not in sys.path:
        sys.path.insert(0, _sparktts_path)
    from sparktts.models.audio_tokenizer import BiCodecTokenizer

from veena3modal.core.constants import (
    BICODEC_TOKENIZER_PATH,
    AUDIO_SAMPLE_RATE,
)

class BiCodecDecoder:
    """
    BiCodec Decoder for Spark TTS.

    Decodes semantic and global tokens to audio waveforms using BiCodec.
    This replaces the SNAC decoder used in the old Indic Orpheus model.

    Supports streaming with sliding window approach (like SNAC).
    """

    def __init__(
        self,
        device: str = "cuda",
        model_path: str = BICODEC_TOKENIZER_PATH,
        enable_batching: bool = False,  # For compatibility with streaming pipeline
    ):
        """
        Initialize BiCodec decoder.

        Args:
            device: Device for BiCodec model (cuda/cpu)
            model_path: Path to BiCodec model checkpoint
            enable_batching: Enable async batching for streaming (not used yet)
        """
        self.device = torch.device(device if torch.cuda.is_available() else "cpu")
        self.model_path = model_path
        self.enable_batching = enable_batching  # For streaming pipeline compatibility

        print(f"🎵 Loading BiCodec audio tokenizer from {model_path}...")
        print(f"   Device: {self.device}")

        # Initialize BiCodec tokenizer
        self.audio_tokenizer = BiCodecTokenizer(model_path, device=str(self.device))

        # OPTIMIZATION: Ensure model is on device once at init, not per-decode call
        self.audio_tokenizer.device = self.device
        self.audio_tokenizer.model.to(self.device)

        # NOTE: torch.compile disabled for BiCodec -- dynamic semantic sequence lengths
        # cause repeated recompilation (~120s each), worse than eager mode.
        # torch.compile works best with static shapes. If needed, pre-warm with
        # representative lengths or use mode="default" instead of "reduce-overhead".
        # The multi-engine approach (Tier 3) provides more impactful gains.

        print(f"✅ BiCodec decoder initialized (sample rate: {AUDIO_SAMPLE_RATE}Hz)")
        if enable_batching:
            print(f"   ℹ  Batching enabled (for streaming support)")

    def decode(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
    ) -> Optional[np.ndarray]:
        """
        Decode BiCodec tokens to audio waveform.

        CRITICAL: BiCodec expects EXACTLY 32 global tokens!
        The decoder pools them via speaker_encoder → d_vector, then broadcasts
        via d_vector.unsqueeze(-1) across the time dimension.

        Args:
            semantic_ids: List of semantic token IDs (variable length)
            global_ids: List of global token IDs (MUST be exactly 32!)

        Returns:
            Audio waveform as numpy array (float32, 16kHz mono)
            Shape: (samples,)
            Returns None if decode fails
        """
        if not semantic_ids or not global_ids:
            print(f"⚠  Empty token lists: semantic={len(semantic_ids)}, global={len(global_ids)}")
            return None

        # Validate global token count
        EXPECTED_GLOBAL_TOKENS = 32
        if len(global_ids) != EXPECTED_GLOBAL_TOKENS:
            print(f"❌ BiCodec requires EXACTLY {EXPECTED_GLOBAL_TOKENS} global tokens, got {len(global_ids)}")
            # Try to fix it
            if len(global_ids) > EXPECTED_GLOBAL_TOKENS:
                global_ids = global_ids[:EXPECTED_GLOBAL_TOKENS]
                print(f"   └─ Truncated to {EXPECTED_GLOBAL_TOKENS}")
            else:
                global_ids = global_ids + [0] * (EXPECTED_GLOBAL_TOKENS - len(global_ids))
                print(f"   └─ Padded to {EXPECTED_GLOBAL_TOKENS}")

        try:
            # Convert to tensors following official Spark-TTS implementation
            # global_token_ids shape: (1, 32) - ALWAYS 32!
            # pred_semantic_ids shape: (1, N) - variable length
            pred_semantic = torch.tensor(semantic_ids).long().unsqueeze(0).to(self.device)
            pred_global = torch.tensor(global_ids).long().unsqueeze(0).to(self.device)

            # Device migration done once in __init__, not per-call
            # Decode through BiCodec
            # BiCodecTokenizer.detokenize() expects:
            #   - global_tokens: (batch, global_dim) -> will unsqueeze(1) internally
            #   - semantic_tokens: (batch, latent_dim)
            with torch.inference_mode():
                wav_np = self.audio_tokenizer.detokenize(
                    pred_global,  # (1, seq) - correct shape!
                    pred_semantic  # (1, seq)
                )

            return wav_np

        except Exception as e:
            print(f"❌ BiCodec decode error: {e}")
            import traceback
            traceback.print_exc()
            return None

    def decode_to_bytes(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
    ) -> Optional[bytes]:
        """
        Decode BiCodec tokens to audio bytes (int16 PCM).

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs

        Returns:
            Audio as bytes (int16 PCM, 16kHz mono)
            Returns None if decode fails
        """
        audio = self.decode(semantic_ids, global_ids)

        if audio is None:
            return None

        # Clip and convert float32 to int16 PCM
        audio = np.clip(audio, -1.0, 1.0)
        audio_int16 = (audio * 32767).astype(np.int16)

        return audio_int16.tobytes()

    def validate_tokens(self, semantic_ids: List[int], global_ids: List[int]) -> bool:
        """
        Validate BiCodec tokens before decoding.

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs

        Returns:
            True if valid, False otherwise
        """
        # Check minimum length
        if not semantic_ids:
            print(f"❌ No semantic tokens")
            return False

        if not global_ids:
            print(f"❌ No global tokens")
            return False

        # Basic sanity checks
        if len(semantic_ids) < 1:
            print(f"❌ Too few semantic tokens: {len(semantic_ids)}")
            return False

        if len(global_ids) < 1:
            print(f"❌ Too few global tokens: {len(global_ids)}")
            return False

        return True

    def get_audio_duration(self, semantic_ids: List[int]) -> float:
        """
        Estimate audio duration from semantic tokens.

        Args:
            semantic_ids: List of semantic token IDs

        Returns:
            Estimated duration in seconds
        """
        # BiCodec generates audio based on semantic tokens
        # Approximate duration calculation (may need tuning)
        # This is an estimate and actual duration depends on model output
        estimated_samples = len(semantic_ids) * 320  # Rough estimate
        return estimated_samples / AUDIO_SAMPLE_RATE

    def decode_streaming(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
        use_sliding_window: bool = False,
        trim_warmup: bool = False,
    ) -> Optional[bytes]:
        """
        Decode BiCodec tokens with streaming support.

        CRITICAL: BiCodec decoder expects EXACTLY 32 global tokens always!
        The decoder internally broadcasts d_vector via d_vector.unsqueeze(-1).

        From sparktts/models/bicodec.py line 184-186:
            d_vector = self.speaker_encoder.detokenize(global_tokens)  # Expects 32 tokens
            x = self.prenet(z_q, d_vector)
            x = x + d_vector.unsqueeze(-1)  # Broadcasts across time automatically!

        Args:
            semantic_ids: List of semantic token IDs (variable length, 50 TPS)
            global_ids: List of global token IDs (MUST be exactly 32 tokens!)
            use_sliding_window: If True, return only middle samples
            trim_warmup: Legacy parameter (not used for BiCodec)

        Returns:
            Audio bytes (int16 PCM) or None if decode fails
        """
        # Minimum semantic tokens needed for stable decode
        MIN_SEMANTIC_TOKENS = 8

        if len(semantic_ids) < MIN_SEMANTIC_TOKENS:
            print(f"⚠  Too few semantic tokens ({len(semantic_ids)}) for decode, need >= {MIN_SEMANTIC_TOKENS}")
            return None

        # CRITICAL: Ensure we have exactly 32 global tokens
        EXPECTED_GLOBAL_TOKENS = 32
        if len(global_ids) != EXPECTED_GLOBAL_TOKENS:
            print(f"⚠  WARNING: Got {len(global_ids)} global tokens, expected {EXPECTED_GLOBAL_TOKENS}")
            if len(global_ids) > EXPECTED_GLOBAL_TOKENS:
                # Truncate to first 32
                global_ids = global_ids[:EXPECTED_GLOBAL_TOKENS]
                print(f"   └─ Truncated to {EXPECTED_GLOBAL_TOKENS} tokens")
            else:
                # Pad with zeros if we somehow have fewer
                global_ids = global_ids + [0] * (EXPECTED_GLOBAL_TOKENS - len(global_ids))
                print(f"   └─ Padded to {EXPECTED_GLOBAL_TOKENS} tokens")

        # Decode with EXACTLY 32 global tokens + variable semantic tokens
        # The decoder handles broadcasting internally!
        audio = self.decode(semantic_ids, global_ids)

        if audio is None:
            return None

        # If using sliding window, return only the middle portion
        if use_sliding_window and len(audio) > 4096:
            # Keep middle samples for overlap-add streaming
            # BiCodec produces ~320 samples per semantic token (16kHz / 50 TPS)
            total_samples = len(audio)
            keep_samples = min(4096, total_samples // 2)
            start = (total_samples - keep_samples) // 2
            end = start + keep_samples
            audio = audio[start:end]

        # Convert to bytes
        audio = np.clip(audio, -1.0, 1.0)
        audio_int16 = (audio * 32767).astype(np.int16)

        return audio_int16.tobytes()

    async def decode_single_async(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
        trim_warmup: bool = False,
        use_sliding_window: bool = False,
    ) -> Optional[bytes]:
        """
        Async wrapper for streaming decode - runs in executor to avoid blocking event loop.

        OPTIMIZATION: Previously ran synchronously, blocking all concurrent coroutines
        during GPU decode (~20-50ms). Now uses run_in_executor to unblock the event loop
        so other streams can progress while this decode runs on GPU.

        Args:
            semantic_ids: List of semantic token IDs
            global_ids: List of global token IDs
            trim_warmup: Legacy parameter (not used)
            use_sliding_window: Use sliding window mode

        Returns:
            Audio bytes (int16 PCM) or None
        """
        import asyncio
        loop = asyncio.get_event_loop()
        return await loop.run_in_executor(
            None,  # Default ThreadPoolExecutor
            self.decode_streaming,
            semantic_ids,
            global_ids,


sed -n '1,260p' /home/ubuntu/veenaModal/veena3modal/core/token_utils.py


Output:
Chunk ID: e49504
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 1578
Output:
"""
BiCodec Token Utilities - Incremental Parsing

Provides O(1) per-token parsing instead of O(n) decode + regex scanning.

The key optimization: instead of:
    tokenizer.decode(all_tokens)  # O(n)
    re.findall(entire_text)       # O(n)

We do:
    for new_token_id in new_tokens:  # Only new tokens
        parsed = token_cache.get(token_id)  # O(1) lookup

This eliminates the O(n²) CPU burn in the streaming hot loop.
"""

from typing import Dict, Optional, Tuple, List
import re

class BiCodecTokenParser:
    """
    Caching parser for BiCodec tokens.

    Converts vLLM token IDs to (type, value) tuples efficiently.

    BiCodec tokens in the vocabulary look like:
        - <|bicodec_semantic_123|> -> ("semantic", 123)
        - <|bicodec_global_7|>     -> ("global", 7)

    Usage:
        parser = BiCodecTokenParser(tokenizer)

        # In hot loop (O(1) per token):
        for token_id in new_token_ids:
            result = parser.parse(token_id)
            if result:
                token_type, value = result
                if token_type == "semantic":
                    semantic_buffer.append(value)
                elif token_type == "global":
                    global_buffer.append(value)
    """

    # Regex patterns for parsing token strings
    SEMANTIC_PATTERN = re.compile(r"^<\|bicodec_semantic_(\d+)\|>$")
    GLOBAL_PATTERN = re.compile(r"^<\|bicodec_global_(\d+)\|>$")

    def __init__(self, tokenizer):
        """
        Initialize parser with tokenizer.

        Args:
            tokenizer: HuggingFace tokenizer with BiCodec special tokens
        """
        self.tokenizer = tokenizer
        # Cache: token_id -> Optional[Tuple[str, int]]
        # None means "not a BiCodec token" (cached negative)
        self._cache: Dict[int, Optional[Tuple[str, int]]] = {}

        # Pre-warm cache with known BiCodec token ranges if possible
        # This is optional but can speed up first-pass parsing
        self._prewarm_cache()

    def _prewarm_cache(self) -> None:
        """
        Pre-populate cache with BiCodec tokens from vocabulary.

        This avoids lazy cache misses during streaming.
        """
        try:
            vocab = self.tokenizer.get_vocab()
            for token_str, token_id in vocab.items():
                # Check if it's a BiCodec token
                if token_str.startswith("<|bicodec_"):
                    parsed = self._parse_token_string(token_str)
                    if parsed:
                        self._cache[token_id] = parsed
        except Exception:
            # If vocab access fails, we'll rely on lazy caching
            pass

    def _parse_token_string(self, token_str: str) -> Optional[Tuple[str, int]]:
        """
        Parse a token string into (type, value) tuple.

        Args:
            token_str: Token string like "<|bicodec_semantic_123|>"

        Returns:
            ("semantic", 123) or ("global", 7) or None if not BiCodec token
        """
        # Try semantic pattern
        match = self.SEMANTIC_PATTERN.match(token_str)
        if match:
            return ("semantic", int(match.group(1)))

        # Try global pattern
        match = self.GLOBAL_PATTERN.match(token_str)
        if match:
            return ("global", int(match.group(1)))

        return None

    def parse(self, token_id: int) -> Optional[Tuple[str, int]]:
        """
        Parse a single token ID into (type, value) tuple.

        O(1) for cached tokens, O(1) amortized for uncached.

        Args:
            token_id: vLLM token ID

        Returns:
            ("semantic", value) or ("global", value) or None
        """
        # Check cache first
        if token_id in self._cache:
            return self._cache[token_id]

        # Cache miss - decode and parse
        try:
            token_str = self.tokenizer.decode([token_id], skip_special_tokens=False)
            parsed = self._parse_token_string(token_str.strip())
            self._cache[token_id] = parsed
            return parsed
        except Exception:
            # Mark as non-BiCodec to avoid repeated failures
            self._cache[token_id] = None
            return None

    def parse_incremental(
        self,
        new_token_ids: List[int],
        semantic_buffer: List[int],
        global_buffer: List[int],
    ) -> Tuple[List[int], List[int]]:
        """
        Parse new tokens and append to existing buffers.

        This is the main method for incremental streaming:
        - Only processes NEW tokens (not entire history)
        - Modifies buffers in-place for efficiency
        - Returns the updated buffers (same objects)

        Args:
            new_token_ids: Only the new token IDs since last call
            semantic_buffer: Existing semantic token values (modified in-place)
            global_buffer: Existing global token values (modified in-place)

        Returns:
            Tuple of (semantic_buffer, global_buffer) - same objects, modified
        """
        for token_id in new_token_ids:
            parsed = self.parse(token_id)
            if parsed:
                token_type, value = parsed
                if token_type == "semantic":
                    semantic_buffer.append(value)
                elif token_type == "global":
                    global_buffer.append(value)

        return semantic_buffer, global_buffer

    def get_cache_stats(self) -> Dict[str, int]:
        """
        Return cache statistics for debugging.

        Returns:
            Dict with cache size and breakdown
        """
        total = len(self._cache)
        bicodec_count = sum(1 for v in self._cache.values() if v is not None)
        return {
            "total_cached": total,
            "bicodec_tokens": bicodec_count,
            "non_bicodec_tokens": total - bicodec_count,
        }

def create_bicodec_parser(tokenizer) -> BiCodecTokenParser:
    """
    Factory function to create a BiCodec token parser.

    Args:
        tokenizer: HuggingFace tokenizer

    Returns:
        Configured BiCodecTokenParser instance
    """
    return BiCodecTokenParser(tokenizer)


sed -n '1,360p' /home/ubuntu/veenaModal/scripts/measure_ttfb_detailed.py


Output:
Chunk ID: 737c63
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 2144
Output:
#!/usr/bin/env python3
"""
Measure TTFB (Time To First Byte) with detailed timing breakdown.

This script will test both streaming and non-streaming modes and show
exactly when the first hearable audio byte is actually received.
"""

import requests
import time
import sys

# Configuration
API_URL = "http://localhost:8000/v1/tts/generate"
API_KEY = None  # Will get from command line or use None for local testing

def measure_streaming_ttfb(text="Hello world, this is a test of streaming audio generation.", speaker="lipakshi"):
    """
    Measure TTFB for streaming mode.
    This is the TRUE TTFB - time until first audio byte arrives.
    """
    print("\n" + "="*80)
    print("📊 STREAMING MODE - TRUE TTFB MEASUREMENT")
    print("="*80)

    payload = {
        "text": text,
        "speaker": speaker,
        "stream": True,
        "format": "wav"
    }

    headers = {"Content-Type": "application/json"}
    if API_KEY:
        headers["X-API-Key"] = API_KEY

    # Timestamps
    t0_request_start = time.time()
    print(f"\n⏰ T0 = {0:.3f}s - Request initiated")

    try:
        # Make request with stream=True
        response = requests.post(API_URL, json=payload, headers=headers, stream=True, timeout=30)

        t1_headers_received = time.time() - t0_request_start
        print(f"⏰ T1 = {t1_headers_received:.3f}s - Headers received (HTTP {response.status_code})")

        if response.status_code != 200:
            print(f"❌ Error: {response.status_code}")
            print(response.text)
            return None

        # Print response headers
        print(f"\n📋 Response Headers:")
        for key, value in response.headers.items():
            if key.startswith('X-'):
                print(f"   {key}: {value}")

        # Read first chunk
        first_chunk = None
        chunk_count = 0
        total_bytes = 0

        for chunk in response.iter_content(chunk_size=4096):
            if chunk:
                chunk_count += 1
                total_bytes += len(chunk)

                if first_chunk is None:
                    first_chunk = chunk
                    t2_first_chunk = time.time() - t0_request_start
                    print(f"\n🎵 T2 = {t2_first_chunk:.3f}s - FIRST AUDIO CHUNK RECEIVED")
                    print(f"   └─ Size: {len(chunk)} bytes")
                    print(f"   └─ TTFB (True): {t2_first_chunk:.3f}s = {t2_first_chunk*1000:.0f}ms")

                    # Check if it's WAV header
                    if chunk.startswith(b'RIFF'):
                        print(f"   └─ Content: WAV header (first 44 bytes)")
                        if len(chunk) > 44:
                            print(f"   └─ Audio data starts: {len(chunk)-44} bytes in first chunk")
                    else:
                        print(f"   └─ Content: Raw audio data")

        t3_complete = time.time() - t0_request_start
        print(f"\n✅ T3 = {t3_complete:.3f}s - Stream complete")
        print(f"   └─ Total chunks: {chunk_count}")
        print(f"   └─ Total bytes: {total_bytes:,}")
        print(f"   └─ Duration: ~{total_bytes / (24000 * 2):.2f}s of audio")

        # Calculate TTFB breakdown
        print(f"\n📊 TTFB BREAKDOWN:")
        print(f"   ├─ Headers received:      {t1_headers_received*1000:>7.0f}ms")
        print(f"   ├─ First chunk received:  {t2_first_chunk*1000:>7.0f}ms  ← TRUE TTFB")
        print(f"   └─ Total time:            {t3_complete*1000:>7.0f}ms")

        return {
            'ttfb': t2_first_chunk,
            'total_time': t3_complete,
            'chunks': chunk_count,
            'bytes': total_bytes
        }

    except Exception as e:
        print(f"❌ Error: {e}")
        import traceback
        traceback.print_exc()
        return None

def measure_non_streaming_ttfb(text="Hello world, this is a test.", speaker="lipakshi"):
    """
    Measure TTFB for non-streaming mode.
    Note: In non-streaming, we receive the entire audio at once.
    """
    print("\n" + "="*80)
    print("📊 NON-STREAMING MODE - RESPONSE TIME MEASUREMENT")
    print("="*80)

    payload = {
        "text": text,
        "speaker": speaker,
        "stream": False,
        "format": "wav"
    }

    headers = {"Content-Type": "application/json"}
    if API_KEY:
        headers["X-API-Key"] = API_KEY

    # Timestamps
    t0_request_start = time.time()
    print(f"\n⏰ T0 = {0:.3f}s - Request initiated")

    try:
        response = requests.post(API_URL, json=payload, headers=headers, timeout=30)

        t1_response_complete = time.time() - t0_request_start
        print(f"⏰ T1 = {t1_response_complete:.3f}s - Full response received (HTTP {response.status_code})")

        if response.status_code != 200:
            print(f"❌ Error: {response.status_code}")
            print(response.text)
            return None

        # Print response headers
        print(f"\n📋 Response Headers:")
        server_ttfb = None
        for key, value in response.headers.items():
            if key.startswith('X-'):
                print(f"   {key}: {value}")
                if key == 'X-TTFB-ms':
                    server_ttfb = float(value) / 1000  # Convert to seconds

        audio_bytes = len(response.content)
        audio_duration = audio_bytes / (24000 * 2)  # 24kHz, 16-bit

        print(f"\n🎵 Audio received:")
        print(f"   └─ Size: {audio_bytes:,} bytes")
        print(f"   └─ Duration: ~{audio_duration:.2f}s")

        print(f"\n📊 TIMING ANALYSIS:")
        print(f"   ├─ Client measured (total):  {t1_response_complete*1000:>7.0f}ms")
        if server_ttfb:
            print(f"   ├─ Server reported (gen):    {server_ttfb*1000:>7.0f}ms  ← Generation time")
            network_time = t1_response_complete - server_ttfb
            print(f"   └─ Network overhead:         {network_time*1000:>7.0f}ms")

        print(f"\n   Note: Non-streaming TTFB = full generation time")
        print(f"         (All audio generated before first byte sent)")

        return {
            'total_time': t1_response_complete,
            'server_ttfb': server_ttfb,
            'bytes': audio_bytes,
            'duration': audio_duration
        }

    except Exception as e:
        print(f"❌ Error: {e}")
        import traceback
        traceback.print_exc()
        return None

def main():
    global API_KEY

    if len(sys.argv) > 1:
        API_KEY = sys.argv[1]

    print("🔍 VEENA3 TTS - DETAILED TTFB MEASUREMENT")
    print("=" * 80)
    print(f"Target: {API_URL}")
    print(f"API Key: {'Provided' if API_KEY else 'None (local testing)'}")

    # Test 1: Short text, streaming
    print("\n\n" + "🧪 TEST 1: Short text, STREAMING MODE".center(80, "="))
    streaming_short = measure_streaming_ttfb(
        text="Hello world, this is a test.",
        speaker="lipakshi"
    )

    time.sleep(0.5)  # Brief pause between tests

    # Test 2: Short text, non-streaming
    print("\n\n" + "🧪 TEST 2: Short text, NON-STREAMING MODE".center(80, "="))
    non_streaming_short = measure_non_streaming_ttfb(
        text="Hello world, this is a test.",
        speaker="lipakshi"
    )

    time.sleep(0.5)

    # Test 3: Longer text, streaming
    print("\n\n" + "🧪 TEST 3: Longer text, STREAMING MODE".center(80, "="))
    long_text = "Hello, this is a longer test of the text to speech system. " * 3
    streaming_long = measure_streaming_ttfb(
        text=long_text,
        speaker="reet"
    )

    # Summary
    print("\n\n" + "="*80)
    print("📊 SUMMARY - TTFB COMPARISON")
    print("="*80)

    if streaming_short:
        print(f"\n✅ Streaming (short):    TTFB = {streaming_short['ttfb']*1000:>6.0f}ms")
    if non_streaming_short:
        print(f"✅ Non-streaming (short): Time = {non_streaming_short['total_time']*1000:>6.0f}ms (full generation)")
    if streaming_long:
        print(f"✅ Streaming (long):     TTFB = {streaming_long['ttfb']*1000:>6.0f}ms")

    print(f"\n💡 KEY INSIGHT:")
    print(f"   • Streaming TTFB = Time to first audio chunk (TRUE TTFB)")
    print(f"   • Non-streaming 'TTFB' = Full generation time (all audio ready)")
    print(f"   • For low latency, use STREAMING mode")
    print("="*80)

if __name__ == "__main__":
    main()


rg -n "VEENA3_TIMING_LOGS|detailed timing|_timing_enabled|logger.info\(\"tts_" /home/ubuntu/veenaModal/veena3modal -g'*.py'


Output:
Chunk ID: acdfb0
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 519
Output:
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:28:_PROFILE_REQUEST_TIMING = "VEENA3_TIMING_LOGS"
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:33:def _timing_enabled() -> bool:
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:441:        if _timing_enabled():
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:494:        if _timing_enabled():
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:495:            logger.info("tts_runtime_nonstream_complete", extra=metrics)
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:504:        if _timing_enabled():
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:622:        if _timing_enabled():
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:690:        if _timing_enabled():
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:692:            logger.info("tts_runtime_chunking_complete", extra=metrics)
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:769:    if _timing_enabled():
/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:866:        if _timing_enabled():
/home/ubuntu/veenaModal/veena3modal/core/pipeline.py:30:def _detailed_timing_enabled() -> bool:
/home/ubuntu/veenaModal/veena3modal/core/pipeline.py:35:      VEENA3_TIMING_LOGS=true|1
/home/ubuntu/veenaModal/veena3modal/core/pipeline.py:37:    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
/home/ubuntu/veenaModal/veena3modal/core/pipeline.py:118:        emit_timing = return_timing or _detailed_timing_enabled()
/home/ubuntu/veenaModal/veena3modal/core/pipeline.py:281:            logger.info("tts_llm_complete", extra=timing_metrics)
/home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:50:def _detailed_timing_enabled() -> bool:
/home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:52:    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
/home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:257:        emit_timing = _detailed_timing_enabled()


sed -n '1,260p' /home/ubuntu/veenaModal/veena3modal/app.py