Output:
Chunk ID: 17f886
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2238
Output:
veena3modal/local_server.py:207:    from veena3modal.services.tts_runtime import initialize_runtime
veena3modal/services/tts_runtime.py:28:_PROFILE_REQUEST_TIMING = "VEENA3_TIMING_LOGS"
veena3modal/services/tts_runtime.py:425:            "tts_runtime_nonstream_start",
veena3modal/services/tts_runtime.py:468:                "llm_batch_count": pipeline_timings.get("batch_count", 0),
veena3modal/services/tts_runtime.py:469:                "llm_batch_gpu_ms_total": pipeline_timings.get("llm_gpu_batch_ms_total", 0),
veena3modal/services/tts_runtime.py:470:                "llm_batch_gpu_ms_min": pipeline_timings.get("llm_gpu_batch_ms_min", 0),
veena3modal/services/tts_runtime.py:471:                "llm_batch_gpu_ms_max": pipeline_timings.get("llm_gpu_batch_ms_max", 0),
veena3modal/services/tts_runtime.py:472:                "llm_batch_gpu_ms_p50": pipeline_timings.get("llm_gpu_batch_ms_p50", 0),
veena3modal/services/tts_runtime.py:511:            logger.info("tts_runtime_nonstream_complete", extra=metrics)
veena3modal/services/tts_runtime.py:522:                "tts_runtime_nonstream_inflight",
veena3modal/services/tts_runtime.py:648:                "tts_runtime_chunking_start",
veena3modal/services/tts_runtime.py:729:                "llm_batch_count": chunk_timing.get("batch_count", 0),
veena3modal/services/tts_runtime.py:730:                "llm_batch_gpu_ms_total": chunk_timing.get("llm_gpu_batch_ms_total", 0),
veena3modal/services/tts_runtime.py:731:                "llm_batch_wall_ms_total": chunk_timing.get("llm_wall_ms", metrics["generation_ms"]),
veena3modal/services/tts_runtime.py:734:                "llm_batch_wall_ms_min": chunk_timing.get("llm_batch_wall_ms_min", 0),
veena3modal/services/tts_runtime.py:735:                "llm_batch_wall_ms_max": chunk_timing.get("llm_batch_wall_ms_max", 0),
veena3modal/services/tts_runtime.py:736:                "llm_batch_gpu_ms_min": chunk_timing.get("llm_batch_gpu_ms_min", 0),
veena3modal/services/tts_runtime.py:737:                "llm_batch_gpu_ms_max": chunk_timing.get("llm_batch_gpu_ms_max", 0),
veena3modal/services/tts_runtime.py:738:                "llm_batch_gpu_ms_p50": chunk_timing.get("llm_batch_gpu_ms_p50", 0),
veena3modal/services/tts_runtime.py:742:                "llm_batch_tokens": chunk_timing.get("token_batches", []),
veena3modal/services/tts_runtime.py:772:            logger.info("tts_runtime_chunking_complete", extra=metrics)
veena3modal/services/tts_runtime.py:851:            "tts_runtime_stream_start",
veena3modal/services/tts_runtime.py:948:                "tts_runtime_stream_end",
veena3modal/core/pipeline.py:36:      VEENA3_TIMING_LOGS=true|1
veena3modal/core/pipeline.py:38:    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
veena3modal/core/pipeline.py:175:                    "tts_llm_batch",
veena3modal/core/pipeline.py:181:                        "llm_batch_wall_ms": round(batch_ms, 3),
veena3modal/core/pipeline.py:182:                        "llm_batch_gpu_ms": round(batch_gpu_ms_value, 3)
veena3modal/core/dual_engine.py:69:    To activate, modify tts_runtime.py to use DualEngineRouter instead of
veena3modal/core/dual_engine.py:118:# 1. In tts_runtime.py, replace:
veena3modal/core/streaming_pipeline.py:54:    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
veena3modal/core/streaming_pipeline.py:470:                                        "tts_stream_decode_batch",
veena3modal/core/streaming_pipeline.py:539:                "llm_batch_count": len(batch_wall_ms),
veena3modal/core/streaming_pipeline.py:540:                "llm_batch_tokens": batch_token_counts,
veena3modal/core/streaming_pipeline.py:541:                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
veena3modal/core/streaming_pipeline.py:542:                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
veena3modal/core/streaming_pipeline.py:543:                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
veena3modal/core/streaming_pipeline.py:544:                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
veena3modal/core/streaming_pipeline.py:545:                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
veena3modal/core/streaming_pipeline.py:546:                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
veena3modal/core/streaming_pipeline.py:547:                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
veena3modal/core/streaming_pipeline.py:548:                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
veena3modal/core/streaming_pipeline.py:569:            logger.info("tts_stream_indic_complete", extra=request_metrics)
veena3modal/core/streaming_pipeline.py:785:        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)
veena3modal/core/streaming_pipeline.py:998:        logger.info("tts_stream_continuation_complete", extra=request_metrics)
veena3modal/app.py:223:            from veena3modal.services.tts_runtime import initialize_runtime
veena3modal/tests/integration/test_tts_generate.py:56:    from veena3modal.services import tts_runtime
veena3modal/tests/integration/test_tts_generate.py:59:    if not tts_runtime.is_initialized():
veena3modal/tests/integration/test_tts_generate.py:61:            tts_runtime.initialize_runtime()
veena3modal/tests/performance/test_benchmarks.py:24:        from veena3modal.services import tts_runtime
veena3modal/tests/performance/test_benchmarks.py:25:        if not tts_runtime.is_initialized():
veena3modal/tests/performance/test_benchmarks.py:42:    from veena3modal.services import tts_runtime
veena3modal/tests/performance/test_benchmarks.py:44:    if not tts_runtime.is_initialized():
veena3modal/tests/performance/test_benchmarks.py:47:    return tts_runtime.get_runtime()
veena3modal/tests/performance/test_benchmarks.py:62:        from veena3modal.services import tts_runtime
veena3modal/tests/performance/test_benchmarks.py:67:        audio_bytes, metrics = await tts_runtime.generate_speech(
veena3modal/tests/performance/test_benchmarks.py:80:        from veena3modal.services import tts_runtime
veena3modal/tests/performance/test_benchmarks.py:87:        async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
veena3modal/tests/performance/test_benchmarks.py:113:        from veena3modal.services import tts_runtime
veena3modal/tests/performance/test_benchmarks.py:118:        audio_bytes, metrics = await tts_runtime.generate_speech(
veena3modal/tests/performance/test_benchmarks.py:133:        from veena3modal.services import tts_runtime
veena3modal/tests/performance/test_benchmarks.py:145:        audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
veena3modal/tests/performance/test_benchmarks.py:170:        from veena3modal.services import tts_runtime
veena3modal/tests/performance/test_benchmarks.py:182:            audio_bytes, metrics = await tts_runtime.generate_speech(
veena3modal/tests/performance/test_benchmarks.py:232:        from veena3modal.services import tts_runtime
veena3modal/tests/performance/test_benchmarks.py:239:            await tts_runtime.generate_speech(
veena3modal/api/fastapi_app.py:120:        from veena3modal.services import tts_runtime
veena3modal/api/fastapi_app.py:121:        model_loaded = tts_runtime.is_initialized()
veena3modal/api/fastapi_app.py:124:        runtime = tts_runtime.get_runtime()
veena3modal/api/fastapi_app.py:182:        from veena3modal.services import tts_runtime
veena3modal/api/fastapi_app.py:263:        if not tts_runtime.is_initialized():
veena3modal/api/fastapi_app.py:317:                audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
veena3modal/api/fastapi_app.py:330:                audio_bytes, metrics = await tts_runtime.generate_speech(
veena3modal/api/fastapi_app.py:361:            runtime = tts_runtime.get_runtime()
veena3modal/api/fastapi_app.py:509:        from veena3modal.services import tts_runtime
veena3modal/api/fastapi_app.py:512:        runtime = tts_runtime.get_runtime()
veena3modal/api/fastapi_app.py:550:                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
veena3modal/api/websocket_handler.py:179:    from veena3modal.services import tts_runtime
veena3modal/api/websocket_handler.py:279:        if not tts_runtime.is_initialized():
veena3modal/api/websocket_handler.py:288:        runtime = tts_runtime.get_runtime()
veena3modal/api/websocket_handler.py:342:            async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
veena3modal/processing/long_text_processor.py:131:        This method is used by _stream_chunked_text in tts_runtime.
scripts/profile_pipeline.py:658:    from veena3modal.services.tts_runtime import initialize_runtime, get_runtime, is_initialized


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '1,320p' veena3modal/tests/performance/test_benchmarks.py


Output:
Chunk ID: e81d31
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2611
Output:
"""
Performance benchmarks for TTS service.

Tests marked with @pytest.mark.slow for CI filtering.
These tests require GPU and model to be loaded.

Run with: pytest veena3modal/tests/performance -v -m slow
"""

import pytest
import time
import asyncio
from typing import List, Tuple

# Skip all tests if model not available
def _can_run_benchmarks() -> Tuple[bool, str]:
    """Check if benchmarks can run."""
    try:
        import torch
        if not torch.cuda.is_available():
            return False, "CUDA not available"

        from veena3modal.services import tts_runtime
        if not tts_runtime.is_initialized():
            return False, "TTS runtime not initialized"

        return True, ""
    except Exception as e:
        return False, str(e)

CAN_RUN, SKIP_REASON = _can_run_benchmarks()

@pytest.fixture(scope="module")
def initialized_runtime():
    """Ensure runtime is initialized for benchmarks."""
    if not CAN_RUN:
        pytest.skip(f"Benchmarks skipped: {SKIP_REASON}")

    from veena3modal.services import tts_runtime

    if not tts_runtime.is_initialized():
        pytest.skip("TTS runtime not initialized")

    return tts_runtime.get_runtime()

@pytest.mark.slow
@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Benchmark prerequisites not met")
class TestTTFBBenchmarks:
    """
    Time-to-First-Byte benchmarks.

    Target: < 500ms warm, < 1200ms cold
    """

    @pytest.mark.asyncio
    async def test_ttfb_short_text_nonstreaming(self, initialized_runtime):
        """TTFB for short text non-streaming."""
        from veena3modal.services import tts_runtime

        text = "Hello, this is a test."

        start = time.time()
        audio_bytes, metrics = await tts_runtime.generate_speech(
            text=text,
            speaker="lipakshi",
        )
        ttfb = (time.time() - start) * 1000

        assert audio_bytes is not None
        assert ttfb < 2000, f"TTFB too high: {ttfb:.0f}ms"
        print(f"TTFB (short, non-streaming): {ttfb:.0f}ms")

    @pytest.mark.asyncio
    async def test_ttfb_streaming_first_chunk(self, initialized_runtime):
        """TTFB for streaming (time to first chunk)."""
        from veena3modal.services import tts_runtime

        text = "Hello, this is a streaming test."

        start = time.time()
        first_chunk_time = None

        async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
            text=text,
            speaker="lipakshi",
        ):
            if first_chunk_time is None:
                first_chunk_time = (time.time() - start) * 1000
            # Don't break - let it complete

        assert first_chunk_time is not None
        assert first_chunk_time < 1500, f"Streaming TTFB too high: {first_chunk_time:.0f}ms"
        print(f"TTFB (streaming, first chunk): {first_chunk_time:.0f}ms")

@pytest.mark.slow
@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Benchmark prerequisites not met")
class TestRTFBenchmarks:
    """
    Real-Time Factor benchmarks.

    RTF = generation_time / audio_duration
    Target: < 0.5 single request, < 0.8 under load
    """

    @pytest.mark.asyncio
    async def test_rtf_short_text(self, initialized_runtime):
        """RTF for short text."""
        from veena3modal.services import tts_runtime

        text = "Hello, this is a test sentence for measuring real-time factor."

        start = time.time()
        audio_bytes, metrics = await tts_runtime.generate_speech(
            text=text,
            speaker="lipakshi",
        )
        generation_time = time.time() - start

        audio_duration = metrics.get("audio_duration_seconds", 0)
        if audio_duration > 0:
            rtf = generation_time / audio_duration
            assert rtf < 1.5, f"RTF too high: {rtf:.2f}"
            print(f"RTF (short): {rtf:.2f} ({audio_duration:.1f}s audio in {generation_time:.1f}s)")

    @pytest.mark.asyncio
    async def test_rtf_long_text(self, initialized_runtime):
        """RTF for long text (with chunking)."""
        from veena3modal.services import tts_runtime

        # ~500 chars of text
        text = """
        This is a longer piece of text that will test the chunking capabilities
        of our TTS system. The text normalization should handle numbers like 12345
        and dates like January 15, 2024. It should also handle currency like $99.99
        and technical terms like API and HTTP. Let's see how well it performs with
        this amount of text that should trigger the chunking mechanism.
        """

        start = time.time()
        audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
            text=text,
            speaker="lipakshi",
        )
        generation_time = time.time() - start

        audio_duration = metrics.get("audio_duration_seconds", 0)
        if audio_duration > 0:
            rtf = generation_time / audio_duration
            assert rtf < 2.0, f"RTF too high for long text: {rtf:.2f}"
            print(f"RTF (long, chunked): {rtf:.2f} ({audio_duration:.1f}s audio in {generation_time:.1f}s)")

@pytest.mark.slow
@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Benchmark prerequisites not met")
class TestThroughputBenchmarks:
    """
    Throughput benchmarks.

    Tests concurrent request handling.
    """

    @pytest.mark.asyncio
    async def test_concurrent_requests(self, initialized_runtime):
        """Measure throughput with concurrent requests."""
        from veena3modal.services import tts_runtime

        texts = [
            "First test sentence.",
            "Second test sentence.",
            "Third test sentence.",
            "Fourth test sentence.",
            "Fifth test sentence.",
        ]

        async def generate(text):
            start = time.time()
            audio_bytes, metrics = await tts_runtime.generate_speech(
                text=text,
                speaker="lipakshi",
            )
            return time.time() - start, metrics.get("audio_duration_seconds", 0)

        start = time.time()
        results = await asyncio.gather(*[generate(t) for t in texts])
        total_time = time.time() - start

        gen_times = [r[0] for r in results]
        audio_durations = [r[1] for r in results]

        total_audio = sum(audio_durations)

        print(f"\nConcurrent throughput ({len(texts)} requests):")
        print(f"  Total wall time: {total_time:.1f}s")
        print(f"  Total audio generated: {total_audio:.1f}s")
        print(f"  Avg per-request time: {sum(gen_times)/len(gen_times):.1f}s")
        print(f"  Effective RTF: {total_time/total_audio:.2f}" if total_audio > 0 else "")

@pytest.mark.slow
@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Benchmark prerequisites not met")
class TestMemoryBenchmarks:
    """
    Memory usage benchmarks.
    """

    def test_gpu_memory_baseline(self, initialized_runtime):
        """Check GPU memory usage at rest."""
        import torch

        if not torch.cuda.is_available():
            pytest.skip("CUDA not available")

        allocated = torch.cuda.memory_allocated() / (1024 ** 3)  # GB
        reserved = torch.cuda.memory_reserved() / (1024 ** 3)  # GB

        print(f"\nGPU Memory (baseline):")
        print(f"  Allocated: {allocated:.2f} GB")
        print(f"  Reserved: {reserved:.2f} GB")

        # Should be reasonable for inference
        assert reserved < 40, f"GPU memory too high: {reserved:.2f} GB"

    @pytest.mark.asyncio
    async def test_gpu_memory_after_generation(self, initialized_runtime):
        """Check GPU memory after generation."""
        import torch
        from veena3modal.services import tts_runtime

        if not torch.cuda.is_available():
            pytest.skip("CUDA not available")

        # Generate some audio
        for i in range(3):
            await tts_runtime.generate_speech(
                text=f"Test sentence number {i+1}.",
                speaker="lipakshi",
            )

        allocated = torch.cuda.memory_allocated() / (1024 ** 3)
        reserved = torch.cuda.memory_reserved() / (1024 ** 3)

        print(f"\nGPU Memory (after 3 generations):")
        print(f"  Allocated: {allocated:.2f} GB")
        print(f"  Reserved: {reserved:.2f} GB")

        # Memory should not leak significantly
        assert reserved < 50, f"GPU memory too high after generation: {reserved:.2f} GB"

@pytest.mark.slow
@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Benchmark prerequisites not met")
class TestCacheBenchmarks:
    """
    Cache performance benchmarks.
    """

    def test_rate_limiter_check_performance(self):
        """Rate limiter check should be < 1ms."""
        from veena3modal.api.rate_limiter import RateLimiter

        limiter = RateLimiter(requests_per_minute=1000)

        times = []
        for i in range(100):
            start = time.time()
            limiter.check(f"key_{i % 10}")
            times.append((time.time() - start) * 1000)

        avg_time = sum(times) / len(times)
        max_time = max(times)

        print(f"\nRate limiter check performance:")
        print(f"  Avg: {avg_time:.3f}ms")
        print(f"  Max: {max_time:.3f}ms")

        assert avg_time < 1, f"Rate limiter too slow: {avg_time:.3f}ms avg"

    def test_api_key_cache_performance(self):
        """API key cache lookup should be < 1ms."""
        from veena3modal.api.auth import ApiKeyCache

        cache = ApiKeyCache()

        # Populate cache
        for i in range(100):
            cache.set(f"key_{i}", {"user_id": f"user-{i}", "credits": 100})

        times = []
        for i in range(100):
            start = time.time()
            cache.get(f"key_{i}")
            times.append((time.time() - start) * 1000)

        avg_time = sum(times) / len(times)
        max_time = max(times)

        print(f"\nAPI key cache lookup performance:")
        print(f"  Avg: {avg_time:.3f}ms")
        print(f"  Max: {max_time:.3f}ms")

        assert avg_time < 1, f"Cache lookup too slow: {avg_time:.3f}ms avg"


Summary: auto


sed -n '1,260p' veena3modal/core/token_utils.py


Output:
Chunk ID: f0ecc8
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1578
Output:
"""
BiCodec Token Utilities - Incremental Parsing

Provides O(1) per-token parsing instead of O(n) decode + regex scanning.

The key optimization: instead of:
    tokenizer.decode(all_tokens)  # O(n)
    re.findall(entire_text)       # O(n)

We do:
    for new_token_id in new_tokens:  # Only new tokens
        parsed = token_cache.get(token_id)  # O(1) lookup

This eliminates the O(n²) CPU burn in the streaming hot loop.
"""

from typing import Dict, Optional, Tuple, List
import re

class BiCodecTokenParser:
    """
    Caching parser for BiCodec tokens.

    Converts vLLM token IDs to (type, value) tuples efficiently.

    BiCodec tokens in the vocabulary look like:
        - <|bicodec_semantic_123|> -> ("semantic", 123)
        - <|bicodec_global_7|>     -> ("global", 7)

    Usage:
        parser = BiCodecTokenParser(tokenizer)

        # In hot loop (O(1) per token):
        for token_id in new_token_ids:
            result = parser.parse(token_id)
            if result:
                token_type, value = result
                if token_type == "semantic":
                    semantic_buffer.append(value)
                elif token_type == "global":
                    global_buffer.append(value)
    """

    # Regex patterns for parsing token strings
    SEMANTIC_PATTERN = re.compile(r"^<\|bicodec_semantic_(\d+)\|>$")
    GLOBAL_PATTERN = re.compile(r"^<\|bicodec_global_(\d+)\|>$")

    def __init__(self, tokenizer):
        """
        Initialize parser with tokenizer.

        Args:
            tokenizer: HuggingFace tokenizer with BiCodec special tokens
        """
        self.tokenizer = tokenizer
        # Cache: token_id -> Optional[Tuple[str, int]]
        # None means "not a BiCodec token" (cached negative)
        self._cache: Dict[int, Optional[Tuple[str, int]]] = {}

        # Pre-warm cache with known BiCodec token ranges if possible
        # This is optional but can speed up first-pass parsing
        self._prewarm_cache()

    def _prewarm_cache(self) -> None:
        """
        Pre-populate cache with BiCodec tokens from vocabulary.

        This avoids lazy cache misses during streaming.
        """
        try:
            vocab = self.tokenizer.get_vocab()
            for token_str, token_id in vocab.items():
                # Check if it's a BiCodec token
                if token_str.startswith("<|bicodec_"):
                    parsed = self._parse_token_string(token_str)
                    if parsed:
                        self._cache[token_id] = parsed
        except Exception:
            # If vocab access fails, we'll rely on lazy caching
            pass

    def _parse_token_string(self, token_str: str) -> Optional[Tuple[str, int]]:
        """
        Parse a token string into (type, value) tuple.

        Args:
            token_str: Token string like "<|bicodec_semantic_123|>"

        Returns:
            ("semantic", 123) or ("global", 7) or None if not BiCodec token
        """
        # Try semantic pattern
        match = self.SEMANTIC_PATTERN.match(token_str)
        if match:
            return ("semantic", int(match.group(1)))

        # Try global pattern
        match = self.GLOBAL_PATTERN.match(token_str)
        if match:
            return ("global", int(match.group(1)))

        return None

    def parse(self, token_id: int) -> Optional[Tuple[str, int]]:
        """
        Parse a single token ID into (type, value) tuple.

        O(1) for cached tokens, O(1) amortized for uncached.

        Args:
            token_id: vLLM token ID

        Returns:
            ("semantic", value) or ("global", value) or None
        """
        # Check cache first
        if token_id in self._cache:
            return self._cache[token_id]

        # Cache miss - decode and parse
        try:
            token_str = self.tokenizer.decode([token_id], skip_special_tokens=False)
            parsed = self._parse_token_string(token_str.strip())
            self._cache[token_id] = parsed
            return parsed
        except Exception:
            # Mark as non-BiCodec to avoid repeated failures
            self._cache[token_id] = None
            return None

    def parse_incremental(
        self,
        new_token_ids: List[int],
        semantic_buffer: List[int],
        global_buffer: List[int],
    ) -> Tuple[List[int], List[int]]:
        """
        Parse new tokens and append to existing buffers.

        This is the main method for incremental streaming:
        - Only processes NEW tokens (not entire history)
        - Modifies buffers in-place for efficiency
        - Returns the updated buffers (same objects)

        Args:
            new_token_ids: Only the new token IDs since last call
            semantic_buffer: Existing semantic token values (modified in-place)
            global_buffer: Existing global token values (modified in-place)

        Returns:
            Tuple of (semantic_buffer, global_buffer) - same objects, modified
        """
        for token_id in new_token_ids:
            parsed = self.parse(token_id)
            if parsed:
                token_type, value = parsed
                if token_type == "semantic":
                    semantic_buffer.append(value)
                elif token_type == "global":
                    global_buffer.append(value)

        return semantic_buffer, global_buffer

    def get_cache_stats(self) -> Dict[str, int]:
        """
        Return cache statistics for debugging.

        Returns:
            Dict with cache size and breakdown
        """
        total = len(self._cache)
        bicodec_count = sum(1 for v in self._cache.values() if v is not None)
        return {
            "total_cached": total,
            "bicodec_tokens": bicodec_count,
            "non_bicodec_tokens": total - bicodec_count,
        }

def create_bicodec_parser(tokenizer) -> BiCodecTokenParser:
    """
    Factory function to create a BiCodec token parser.

    Args:
        tokenizer: HuggingFace tokenizer

    Returns:
        Configured BiCodecTokenParser instance
    """
    return BiCodecTokenParser(tokenizer)


sed -n '320,700p' veena3modal/local_server.py


Output:
Chunk ID: a921bd
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 1129
Output:
        "--enable-sr",
        action="store_true",
        default=False,
        help="Enable AP-BWE super-resolution (16kHz -> 48kHz)",
    )
    sr_group.add_argument(
        "--sr-path",
        type=str,
        default=os.environ.get("AP_BWE_CHECKPOINT_DIR"),
        help="Path to AP-BWE checkpoint directory",
    )

    # Server configuration
    server_group = parser.add_argument_group("Server")
    server_group.add_argument(
        "--host",
        type=str,
        default="0.0.0.0",
        help="Bind address (default: 0.0.0.0)",
    )
    server_group.add_argument(
        "--port",
        type=int,
        default=int(os.environ.get("PORT", "8000")),
        help="Port to serve on (default: 8000)",
    )
    server_group.add_argument(
        "--workers",
        type=int,
        default=1,
        help="Number of uvicorn workers (default: 1, each loads full model)",
    )
    server_group.add_argument(
        "--reload",
        action="store_true",
        default=False,
        help="Enable auto-reload for development (incompatible with --workers > 1)",
    )

    # Auth
    auth_group = parser.add_argument_group("Auth")
    auth_group.add_argument(
        "--auth",
        action="store_true",
        default=False,
        help="Enable API key authentication (default: disabled for local)",
    )

    # Logging
    log_group = parser.add_argument_group("Logging")
    log_group.add_argument(
        "--log-level",
        type=str,
        default="INFO",
        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
        help="Log level (default: INFO)",
    )

    return parser.parse_args()

def main():
    """
    Main entry point for local TTS server.

    Lifecycle:
    1. Parse args + configure logging
    2. Download model if needed
    3. Initialize TTS runtime (vLLM engine, decoders, pipelines)
    4. Create FastAPI app
    5. Run uvicorn
    """
    args = parse_args()

    # Configure logging
    logging.basicConfig(
        level=getattr(logging, args.log_level),
        format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )

    # Auth bypass for local development (unless --auth flag is set)
    if not args.auth:
        os.environ["AUTH_BYPASS_MODE"] = "true"
        logger.info("Auth bypass enabled (local mode). Use --auth to require API keys.")
    else:
        os.environ.setdefault("AUTH_BYPASS_MODE", "false")

    # Resolve model path
    # Spark TTS root has config.yaml; LLM subdir has config.json; fine-tuned may have config.json at root
    model_path = args.model_path
    model_exists = any(
        os.path.exists(os.path.join(model_path, marker))
        for marker in ("config.json", "config.yaml", "LLM/config.json")
    )
    if not model_exists:
        if args.skip_download:
            logger.error(f"Model not found at {model_path} and --skip-download is set")
            sys.exit(1)
        model_path = download_model(
            hf_repo=args.hf_repo,
            local_dir=model_path,
            hf_token=args.hf_token,
        )

    # Initialize TTS runtime (this is the expensive step - loads model onto GPU)
    initialize_local_runtime(
        model_path=model_path,
        device=args.device,
        gpu_memory_utilization=args.gpu_memory,
        enable_sr=args.enable_sr,
        sr_checkpoint_dir=args.sr_path,
        hf_token=args.hf_token,
        num_engines=args.num_engines,
    )

    # Run uvicorn
    # NOTE: We use the factory string so uvicorn can find the app.
    # With --reload, uvicorn reimports, but the runtime singleton persists in memory.
    import uvicorn

    logger.info("")
    logger.info("=" * 60)
    logger.info(f"Veena3 TTS Local Server starting")
    logger.info(f"  URL: http://{args.host}:{args.port}")
    logger.info(f"  Health: http://{args.host}:{args.port}/v1/tts/health")
    logger.info(f"  Generate: POST http://{args.host}:{args.port}/v1/tts/generate")
    logger.info(f"  Metrics: http://{args.host}:{args.port}/v1/tts/metrics")
    logger.info(f"  Docs: http://{args.host}:{args.port}/docs")
    logger.info("=" * 60)

    uvicorn.run(
        "veena3modal.local_server:create_local_app",
        factory=True,
        host=args.host,
        port=args.port,
        workers=args.workers,
        reload=args.reload,
        log_level=args.log_level.lower(),
        # Performance: keep-alive for connection reuse under load
        timeout_keep_alive=30,
    )

if __name__ == "__main__":
    main()


rg -n "initialize_runtime|generate_speech_streaming\(|veena3modal/services/tts_runtime|stress" scripts veena3modal | head -n 200


Output:
Chunk ID: 04361f
Wall time: 0.0531 seconds
Process exited with code 0
Original token count: 628
Output:
veena3modal/local_server.py:207:    from veena3modal.services.tts_runtime import initialize_runtime
veena3modal/local_server.py:224:    runtime = initialize_runtime(
veena3modal/app.py:223:            from veena3modal.services.tts_runtime import initialize_runtime
veena3modal/app.py:226:            self.runtime = initialize_runtime(
veena3modal/services/tts_runtime.py:92:def initialize_runtime(
veena3modal/services/tts_runtime.py:785:async def generate_speech_streaming(
scripts/stress_test_production.py:88:# === GPU Monitor (reused from stress_test_local.py) ===
scripts/stress_test_production.py:406:    parser.add_argument("--output", default="stress_test_production.json")
veena3modal/tests/integration/test_tts_generate.py:61:            tts_runtime.initialize_runtime()
scripts/profile_pipeline.py:658:    from veena3modal.services.tts_runtime import initialize_runtime, get_runtime, is_initialized
scripts/profile_pipeline.py:661:        initialize_runtime(
veena3modal/api/fastapi_app.py:550:                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
veena3modal/api/websocket_handler.py:342:            async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
veena3modal/tests/modal_live/test_load.py:307:        """50 concurrent requests (stress test)."""
veena3modal/tests/modal_live/test_load.py:311:            text="High concurrency stress test.",
scripts/stress_test_local.py:16:    python scripts/stress_test_local.py
scripts/stress_test_local.py:19:    python scripts/stress_test_local.py --quick
scripts/stress_test_local.py:22:    python scripts/stress_test_local.py --levels 1,5,10,20
scripts/stress_test_local.py:25:    python scripts/stress_test_local.py --stream-only
scripts/stress_test_local.py:28:    python scripts/stress_test_local.py --no-stream
scripts/stress_test_local.py:31:    python scripts/stress_test_local.py --url http://localhost:8080
scripts/stress_test_local.py:53:# Test texts at different lengths to stress different parts of the pipeline
scripts/stress_test_local.py:545:async def run_stress_test(args):
scripts/stress_test_local.py:546:    """Run the full stress test suite."""
scripts/stress_test_local.py:670:    parser.add_argument("--output", default="stress_test_results.json", help="Output JSON path")
scripts/stress_test_local.py:677:    asyncio.run(run_stress_test(args))
veena3modal/tests/performance/test_benchmarks.py:87:        async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(


sed -n '560,860p' scripts/profile_pipeline.py


Output:
Chunk ID: eaf1b9
Wall time: 0.0502 seconds
Process exited with code 0
Original token count: 1586
Output:
        else:
            return await profile_non_streaming(runtime, text, spk)

    profiles = await asyncio.gather(*[run_one(i) for i in range(num_requests)])
    return list(profiles)

def print_concurrency_summary(profiles: List[RequestProfile], concurrency: int, stream: bool):
    """Print aggregated summary for concurrent profiles."""
    import statistics

    mode = "streaming" if stream else "non-streaming"
    totals = [p.total_ms for p in profiles]
    wall_time = max(totals)

    print(f"\n{'#' * 80}")
    print(f"  CONCURRENCY={concurrency} ({mode}), {len(profiles)} requests")
    print(f"{'#' * 80}")
    print(f"  Wall time: {wall_time:.0f}ms")
    print(f"  Throughput: {len(profiles) / (wall_time / 1000):.2f} req/s")
    print(f"  Latency: avg={statistics.mean(totals):.0f}ms  "
          f"p50={sorted(totals)[len(totals)//2]:.0f}ms  "
          f"p99={sorted(totals)[int(len(totals)*0.99)]:.0f}ms")

    if stream:
        ttfbs = [p.ttfb_ms for p in profiles if p.ttfb_ms > 0]
        if ttfbs:
            print(f"  TTFB: avg={statistics.mean(ttfbs):.0f}ms  "
                  f"p50={sorted(ttfbs)[len(ttfbs)//2]:.0f}ms  "
                  f"p95={sorted(ttfbs)[int(len(ttfbs)*0.95)]:.0f}ms")

    # Aggregate stage timing
    stage_totals: Dict[str, List[float]] = {}
    for p in profiles:
        for name, s in p.stages.items():
            if name not in stage_totals:
                stage_totals[name] = []
            stage_totals[name].append(s.total_ms)

    print(f"\n  STAGE BREAKDOWN (avg across {len(profiles)} requests):")
    print(f"  {'-' * 76}")
    sorted_stages = sorted(stage_totals.items(), key=lambda x: -statistics.mean(x[1]))
    for name, times in sorted_stages:
        avg = statistics.mean(times)
        mx = max(times)
        mn = min(times)
        pct = (avg / statistics.mean(totals)) * 100 if totals else 0
        print(f"  {name:<35} avg={avg:>8.2f}ms  min={mn:>8.2f}ms  max={mx:>8.2f}ms  ({pct:.1f}%)")

    # BiCodec decode contention
    all_decode_lats = []
    for p in profiles:
        all_decode_lats.extend(p.decode_latencies)
    if all_decode_lats:
        print(f"\n  BICODEC DECODE (all calls across {len(profiles)} requests):")
        print(f"    total calls: {len(all_decode_lats)}")
        print(f"    avg={statistics.mean(all_decode_lats):.2f}ms  "
              f"p50={sorted(all_decode_lats)[len(all_decode_lats)//2]:.2f}ms  "
              f"p99={sorted(all_decode_lats)[int(len(all_decode_lats)*0.99)]:.2f}ms  "
              f"max={max(all_decode_lats):.2f}ms")

    # Token generation rates
    all_rates = []
    for p in profiles:
        all_rates.extend(p.token_generation_rates)
    if all_rates:
        print(f"\n  TOKEN GEN RATE (all samples):")
        print(f"    avg={statistics.mean(all_rates):.0f} tok/s  "
              f"min={min(all_rates):.0f}  max={max(all_rates):.0f}")

# === GPU Memory Snapshot ===

def gpu_snapshot() -> str:
    try:
        import torch
        if not torch.cuda.is_available():
            return "No GPU"
        alloc = torch.cuda.memory_allocated() / 1e9
        reserved = torch.cuda.memory_reserved() / 1e9
        total = torch.cuda.get_device_properties(0).total_memory / 1e9
        return f"GPU: {alloc:.1f}GB alloc, {reserved:.1f}GB reserved / {total:.1f}GB total"
    except Exception:
        return "GPU info unavailable"

# === Main ===

async def main_async(args):
    # Initialize runtime
    logger.info("Initializing TTS runtime for profiling...")
    logger.info(gpu_snapshot())

    from veena3modal.local_server import resolve_model_paths, DEFAULT_LOCAL_MODEL_DIR

    model_path = args.model_path
    llm_path, bicodec_path = resolve_model_paths(model_path)

    from veena3modal.services.tts_runtime import initialize_runtime, get_runtime, is_initialized

    if not is_initialized():
        initialize_runtime(
            model_path=llm_path,
            bicodec_path=bicodec_path,
            device="cuda",
            # gpu_memory_utilization uses VLLM_CONFIG default (0.25 after Tier 1 optimization)
        )

    runtime = get_runtime()
    logger.info(f"Runtime ready: {runtime.model_version}")
    logger.info(gpu_snapshot())

    text = TEXTS.get(args.text, TEXTS["short"])
    concurrency_levels = [int(x) for x in args.concurrency.split(",")]

    # === Single Request Profiles ===
    if not args.concurrency_only:
        logger.info(f"\n{'=' * 80}")
        logger.info(f"SINGLE REQUEST PROFILING (text={args.text}, {len(text)} chars)")
        logger.info(f"{'=' * 80}")

        # Non-streaming
        if not args.stream:
            logger.info("\nProfiling non-streaming...")
            p = await profile_non_streaming(runtime, text, "lipakshi")
            print(p.report())

        # Streaming
        if not args.no_stream:
            logger.info("\nProfiling streaming...")
            p = await profile_streaming(runtime, text, "lipakshi")
            print(p.report())

    # === Concurrency Profiles ===
    for conc in concurrency_levels:
        modes = []
        if not args.stream:
            modes.append(False)
        if not args.no_stream:
            modes.append(True)

        for stream_mode in modes:
            logger.info(f"\nProfiling {conc} concurrent {'streaming' if stream_mode else 'non-streaming'} requests...")
            profiles = await profile_concurrent(runtime, conc, text, stream=stream_mode)
            print_concurrency_summary(profiles, conc, stream_mode)

    logger.info(f"\n{gpu_snapshot()}")
    logger.info("Profiling complete.")

def main():
    parser = argparse.ArgumentParser(description="Veena3 TTS Pipeline Profiler")
    parser.add_argument("--model-path", default=os.path.join(REPO_ROOT, "models", "spark_tts_4speaker"))
    parser.add_argument("--text", default="short", choices=["short", "medium", "long"])
    parser.add_argument("--concurrency", default="1,5,10,20", help="Concurrency levels (comma-separated)")
    parser.add_argument("--stream", action="store_true", help="Stream mode only")
    parser.add_argument("--no-stream", action="store_true", help="Non-stream mode only")
    parser.add_argument("--concurrency-only", action="store_true", help="Skip single-request profiles")
    args = parser.parse_args()
    asyncio.run(main_async(args))

if __name__ == "__main__":
    main()


sed -n '1,260p' veena3modal/tests/integration/test_tts_generate.py


Output:
Chunk ID: 6650bd
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 2143
Output:
"""
Integration tests for /v1/tts/generate endpoint (M3).

These tests require the TTS model to be loaded.
Skip gracefully if model/GPU not available.
"""

import os
import pytest

# Check if we can run integration tests
def _can_run_integration_tests():
    """Check if we have the prerequisites for integration tests."""
    try:
        import torch
        if not torch.cuda.is_available():
            return False, "No GPU available"
    except ImportError:
        return False, "PyTorch not installed"

    # Check for model path
    model_path = os.environ.get('SPARK_TTS_MODEL_PATH') or os.environ.get('MODEL_PATH')
    if not model_path:
        # Check default paths
        default_paths = [
            '/models/spark_tts_4speaker',
            '/home/ubuntu/veena3/models/spark_tts_4speaker',
        ]
        for path in default_paths:
            if os.path.exists(path):
                return True, None
        return False, "Model path not found"

    if not os.path.exists(model_path):
        return False, f"Model path does not exist: {model_path}"

    return True, None

CAN_RUN, SKIP_REASON = _can_run_integration_tests()

@pytest.fixture(scope="module")
def initialized_client():
    """
    Create a test client with initialized TTS runtime.

    This fixture initializes the model once per test module.
    """
    if not CAN_RUN:
        pytest.skip(f"Integration tests skipped: {SKIP_REASON}")

    from fastapi.testclient import TestClient
    from veena3modal.api.fastapi_app import create_app
    from veena3modal.services import tts_runtime

    # Initialize runtime if not already done
    if not tts_runtime.is_initialized():
        try:
            tts_runtime.initialize_runtime()
        except Exception as e:
            pytest.skip(f"Could not initialize TTS runtime: {e}")

    app = create_app()
    client = TestClient(app)

    yield client

    # Cleanup (optional - runtime persists for performance)

@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Integration test prerequisites not met")
class TestTTSGenerateEndpoint:
    """Test /v1/tts/generate endpoint with real model."""

    def test_simple_generation(self, initialized_client):
        """Basic non-streaming WAV generation."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "text": "Hello, this is a test.",
                "speaker": "lipakshi",
                "stream": False,
                "format": "wav",
            }
        )

        # Debug: print error response if not 200
        if response.status_code != 200:
            print(f"Error response: {response.status_code} - {response.text}")

        assert response.status_code == 200
        assert response.headers["content-type"] == "audio/wav"
        assert "X-Request-ID" in response.headers
        assert "X-Model-Version" in response.headers
        assert "X-Audio-Bytes" in response.headers
        assert "X-TTFB-ms" in response.headers

        # Check we got actual audio
        audio_bytes = response.content
        assert len(audio_bytes) > 44  # More than just WAV header
        assert audio_bytes[:4] == b'RIFF'  # WAV magic bytes

    def test_generation_with_emotion(self, initialized_client):
        """Generation with emotion tags."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "text": "[laughs] That's so funny!",
                "speaker": "reet",
                "stream": False,
            }
        )

        assert response.status_code == 200
        assert len(response.content) > 44

    def test_generation_friendly_speaker(self, initialized_client):
        """Generation with friendly speaker name."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "text": "Testing friendly speaker name.",
                "speaker": "Mitra",  # Friendly name for lipakshi
                "stream": False,
            }
        )

        assert response.status_code == 200

    def test_generation_with_seed(self, initialized_client):
        """Generation with seed for reproducibility."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "text": "Testing with seed.",
                "speaker": "Nandini",
                "seed": 42,
                "stream": False,
            }
        )

        assert response.status_code == 200
        assert response.headers.get("X-Seed") == "42"

    def test_generation_hindi_text(self, initialized_client):
        """Generation with Hindi text."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "text": "नमस्ते, यह एक परीक्षण है।",
                "speaker": "lipakshi",
                "stream": False,
            }
        )

        assert response.status_code == 200
        assert len(response.content) > 44

    def test_chunking_disabled(self, initialized_client):
        """Generation with chunking disabled."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "text": "Short text without chunking.",
                "speaker": "lipakshi",
                "chunking": False,
                "stream": False,
            }
        )

        assert response.status_code == 200
        assert response.headers.get("X-Text-Chunked") == "false"

@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Integration test prerequisites not met")
class TestTTSGenerateErrors:
    """Test error handling for /v1/tts/generate."""

    def test_missing_text(self, initialized_client):
        """Missing text should return 400."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "speaker": "lipakshi",
            }
        )

        assert response.status_code == 400
        assert "error" in response.json()

    def test_missing_speaker(self, initialized_client):
        """Missing speaker should return 400."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "text": "Hello",
            }
        )

        assert response.status_code == 400
        assert "error" in response.json()

    def test_invalid_speaker(self, initialized_client):
        """Invalid speaker should return 400."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "text": "Hello",
                "speaker": "invalid_speaker_name",
            }
        )

        assert response.status_code == 400

    def test_streaming_returns_audio(self, initialized_client):
        """Streaming should return chunked audio response."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "text": "Hello, this is a streaming test.",
                "speaker": "lipakshi",
                "stream": True,
            }
        )

        # Streaming returns 200 with audio content
        assert response.status_code == 200
        assert response.headers["content-type"] == "audio/wav"
        assert response.headers.get("X-Stream") == "true"

        # Check we got actual audio
        audio_bytes = response.content
        assert len(audio_bytes) > 44  # More than just WAV header
        assert audio_bytes[:4] == b'RIFF'  # WAV magic bytes

    def test_non_wav_format_not_implemented(self, initialized_client):
        """Non-WAV formats should return 501 (not implemented)."""
        response = initialized_client.post(
            "/v1/tts/generate",
            json={
                "text": "Hello",
                "speaker": "lipakshi",
                "format": "opus",
            }
        )

        assert response.status_code == 501
        assert "FORMAT_NOT_IMPLEMENTED" in response.json().get("error", {}).get("code", "")

@pytest.mark.skipif(not CAN_RUN, reason=SKIP_REASON or "Integration test prerequisites not met")
class TestTTSStreaming:
    """Test streaming-specific functionality."""

    def test_streaming_with_emotion(self, initialized_client):
        """Streaming with emotion tags should work."""
        response = initialized_client.post(
            "/v1/tts/generate",


sed -n '1,260p' scripts/stress_test_production.py


Output:
Chunk ID: edb9d6
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2919
Output:
#!/usr/bin/env python3
"""
Production Stress Test - Real-world sentences for Veena3 TTS.

Uses actual production-quality sentences across:
- Multiple languages (English, Hindi, Telugu)
- Various lengths (short greetings to long paragraphs)
- Emotion tags
- Mixed scripts
- Numbers, dates, special characters

Runs the same concurrency ramp as the baseline test for direct comparison.
"""

import argparse
import asyncio
import json
import os
import statistics
import subprocess
import sys
import time
import threading
from dataclasses import dataclass, field
from typing import List, Optional, Dict, Any

import httpx

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

# === Real production sentences ===
# These mimic actual API traffic: mixed languages, emotions, varied lengths

REAL_SENTENCES = {
    "en_short": [
        "Welcome to Veena, your personal voice assistant.",
        "The weather today is sunny with a high of thirty two degrees.",
        "Your order has been confirmed and will arrive by tomorrow.",
        "Please hold on, I am transferring you to a specialist.",
        "Thank you for calling. Have a wonderful day!",
    ],
    "en_medium": [
        "Good morning! I hope you are having a great start to your day. Let me walk you through the key highlights from yesterday's meeting.",
        "The quarterly revenue report shows a fifteen percent increase compared to last year, driven primarily by growth in the South Asian market.",
        "I would like to remind you that your subscription is expiring on March fifteenth. Would you like me to help you with the renewal process?",
        "According to the latest research, artificial intelligence is expected to contribute over fifteen trillion dollars to the global economy by twenty thirty.",
        "[excited] Congratulations! You have been selected for our premium membership program. Let me tell you about the amazing benefits.",
    ],
    "en_long": [
        "In the heart of Hyderabad, where the old city meets the new, there is a small bookshop that has been running for over fifty years. The owner, Ramesh uncle, knows every book by heart. He can tell you exactly which shelf to find your favorite author on, and he always has a recommendation ready. Last Tuesday, a young girl walked in looking for a book on astronomy. Ramesh uncle smiled and led her to the science section.",
        "The Indian space program has achieved remarkable milestones in the past decade. From the Mars Orbiter Mission in twenty fourteen to the Chandrayaan three lunar landing in twenty twenty three, ISRO has consistently demonstrated that world class space exploration does not require a massive budget. The upcoming Gaganyaan mission will mark India's first crewed spaceflight.",
    ],
    "hi_short": [
        "नमस्ते! आज का दिन बहुत अच्छा है।",
        "कृपया अपना नाम और फोन नंबर बताइए।",
        "आपका ऑर्डर सफलतापूर्वक प्लेस हो गया है।",
        "धन्यवाद! आपकी कॉल हमारे लिए महत्वपूर्ण है।",
    ],
    "hi_medium": [
        "[curious] क्या आप जानते हैं कि भारत दुनिया का सबसे बड़ा लोकतंत्र है? यहाँ एक सौ चालीस करोड़ से अधिक लोग रहते हैं।",
        "आज के मौसम का हाल बताते हैं। दिल्ली में तापमान बत्तीस डिग्री रहेगा और हल्की बारिश की संभावना है।",
    ],
    "te_short": [
        "నమస్కారం! మీరు ఎలా ఉన్నారు?",
        "మీ ఆర్డర్ విజయవంతంగా ప్లేస్ చేయబడింది।",
        "దయచేసి కొంచెం ఆగండి, మేము మీకు సహాయం చేస్తాము.",
    ],
    "emotion_mixed": [
        "[laughs] Oh that is so funny! I cannot believe you said that.",
        "[whispers] Let me tell you a secret. The new product launch is happening next week.",
        "[excited] Amazing news everyone! We just crossed one million users!",
        "[sighs] Unfortunately, the flight has been delayed by two hours. I apologize for the inconvenience.",
        "[angry] This is unacceptable! The service was supposed to be restored by noon today.",
    ],
}

# Flatten and tag for round-robin selection
ALL_SENTENCES = []
for category, sentences in REAL_SENTENCES.items():
    for s in sentences:
        ALL_SENTENCES.append({"text": s, "category": category, "length": len(s)})

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika",
            "adarsh", "Nilay", "Aarvi", "Asha", "Bittu", "Mira"]

# === GPU Monitor (reused from stress_test_local.py) ===

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=5)

    def _poll(self):
        while self._running:
            try:
                r = subprocess.run(
                    ["nvidia-smi", "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                     "--format=csv,noheader,nounits"],
                    capture_output=True, text=True, timeout=5)
                if r.returncode == 0:
                    parts = r.stdout.strip().split(", ")
                    if len(parts) >= 4:
                        self.snapshots.append(GPUSnapshot(
                            timestamp=time.time(),
                            memory_used_mb=float(parts[0]),
                            memory_total_mb=float(parts[1]),
                            gpu_utilization_pct=float(parts[2]),
                            temperature_c=float(parts[3]),
                        ))
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self):
        if not self.snapshots:
            return {}
        mem = [s.memory_used_mb for s in self.snapshots]
        util = [s.gpu_utilization_pct for s in self.snapshots]
        return {
            "memory_mb_avg": statistics.mean(mem),
            "memory_mb_peak": max(mem),
            "memory_mb_total": self.snapshots[0].memory_total_mb,
            "gpu_util_avg": statistics.mean(util),
            "gpu_util_peak": max(util),
            "temp_avg": statistics.mean(s.temperature_c for s in self.snapshots),
            "temp_peak": max(s.temperature_c for s in self.snapshots),
        }

# === Request Result ===

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float
    ttfb_ms: float = 0.0
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    error: Optional[str] = None
    text_length: int = 0
    category: str = ""
    stream: bool = False

# === HTTP Client ===

async def make_request(client, base_url, text, speaker, stream=False, timeout=120.0):
    url = f"{base_url}{GENERATE_PATH}"
    payload = {"text": text, "speaker": speaker, "stream": stream, "format": "wav"}
    start = time.time()

    try:
        if stream:
            async with client.stream("POST", url, json=payload,
                                     headers={"Content-Type": "application/json"},
                                     timeout=timeout) as resp:
                first_byte = True
                ttfb = 0.0
                total = 0
                async for chunk in resp.aiter_bytes():
                    if first_byte:
                        ttfb = (time.time() - start) * 1000
                        first_byte = False
                    total += len(chunk)
                latency = (time.time() - start) * 1000
                if resp.status_code == 200:
                    pcm = max(0, total - 44)
                    return RequestResult(True, 200, latency, ttfb, total,
                                        pcm / (16000 * 2), text_length=len(text), stream=True)
                return RequestResult(False, resp.status_code, latency, ttfb,
                                     error=f"HTTP {resp.status_code}", text_length=len(text), stream=True)
        else:
            resp = await client.post(url, json=payload,
                                     headers={"Content-Type": "application/json"},
                                     timeout=timeout)
            latency = (time.time() - start) * 1000
            audio_sec = 0.0
            h = resp.headers.get("X-Audio-Seconds")
            if h:
                audio_sec = float(h)
            elif resp.status_code == 200:
                audio_sec = max(0, len(resp.content) - 44) / (16000 * 2)
            if resp.status_code == 200:
                return RequestResult(True, 200, latency, latency, len(resp.content),
                                     audio_sec, text_length=len(text), stream=False)
            return RequestResult(False, resp.status_code, latency,
                                 error=resp.text[:200], text_length=len(text), stream=False)
    except httpx.TimeoutException:
        return RequestResult(False, 0, (time.time() - start) * 1000, error="TIMEOUT",
                             text_length=len(text), stream=stream)
    except Exception as e:
        return RequestResult(False, 0, (time.time() - start) * 1000, error=str(e)[:200],
                             text_length=len(text), stream=stream)

def percentile(values, pct):
    if not values:
        return 0
    s = sorted(values)
    return s[min(int(len(s) * pct / 100), len(s) - 1)]

async def run_level(base_url, num_requests, concurrency, stream, gpu_monitor=None):
    sem = asyncio.Semaphore(concurrency)
    results = []

    async def req(i, client):
        async with sem:
            item = ALL_SENTENCES[i % len(ALL_SENTENCES)]
            speaker = SPEAKERS[i % len(SPEAKERS)]
            r = await make_request(client, base_url, item["text"], speaker, stream)
            r.category = item["category"]
            return r

    if gpu_monitor:
        gpu_monitor.snapshots.clear()

    start = time.time()
    limits = httpx.Limits(max_connections=concurrency + 10, max_keepalive_connections=concurrency)
    async with httpx.AsyncClient(limits=limits) as client:
        tasks = [req(i, client) for i in range(num_requests)]
        raw = await asyncio.gather(*tasks, return_exceptions=True)
    wall = time.time() - start

    for r in raw:
        if isinstance(r, Exception):
            results.append(RequestResult(False, 0, 0, error=str(r)[:200], stream=stream))
        else:
            results.append(r)

    successes = [r for r in results if r.success]
    failures = [r for r in results if not r.success]
    lats = [r.latency_ms for r in successes]
    ttfbs = [r.ttfb_ms for r in successes if r.ttfb_ms > 0]
    audio_total = sum(r.audio_seconds for r in successes)
    rps = num_requests / wall if wall > 0 else 0


I’m ready to patch the remaining runtime/API wiring and add a dedicated local benchmark script. I’ll also correct a malformed section in the streaming pipeline so it can emit reliable per-request timing during concurrency tests.


Output:
Chunk ID: 0e8029
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 134
Output:
 D CLEANUP_PLAN.md
 D PROGRESS.md
 D migration.md
 M veena3modal/core/bicodec_decoder.py
 M veena3modal/core/constants.py
 M veena3modal/core/pipeline.py
 M veena3modal/core/streaming_pipeline.py
 M veena3modal/services/tts_runtime.py
?? scripts/profile_pipeline.py
?? scripts/setup_local.sh
?? scripts/stress_test_local.py
?? scripts/stress_test_production.py
?? veena3modal/__main__.py
?? veena3modal/core/bicodec_batch_decoder.py
?? veena3modal/core/dual_engine.py
?? veena3modal/core/multi_engine.py
?? veena3modal/local_server.py


Output:
Chunk ID: cd4147
Wall time: 0.0504 seconds
Process exited with code 127
Original token count: 12
Output:
/bin/bash: line 1: python: command not found


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py
@@
         if request_id is None:
             import uuid
             request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
             request_metrics["request_id"] = request_id
@@
         for request_output in results_generator:
             new_token_ids = generated_ids[processed_token_count:]
             processed_token_count = len(generated_ids)

             parse_start = time.time()
             # OPTIMIZATION: Only process NEW tokens (incremental parsing)
@@
             parse_wall_ms = (time.time() - parse_start) * 1000
             parse_wall_ms_list.append(parse_wall_ms)
             batch_token_counts.append(len(new_token_ids))
-            batch_wall_ms.append((time.time() - t_iter) * 1000 if t_iter else 0.0)
+            batch_wall_ms_value = (time.time() - t_iter) * 1000 if t_iter else 0.0
+            batch_wall_ms.append(batch_wall_ms_value)
+
+            if emit_timing:
+                logger.info(
+                    "tts_stream_llm_batch",
+                    extra={
+                        "speaker": speaker,
+                        "request_id": request_id,
+                        "batch_index": len(batch_wall_ms),
+                        "batch_tokens": len(new_token_ids),
+                        "llm_batch_wall_ms": int(batch_wall_ms_value),
+                        "llm_batch_gpu_ms": int(batch_gpu_ms[-1]) if batch_gpu_ms else None,
+                    },
+                )
@@
                                 if emit_timing:
-                                    logger.info(
-                                        "tts_stream_decode_batch",
-                                        extra={
-                                            "speaker": speaker,
-                                            "request_id": request_id,
-                                            "batch_index": decode_calls,
-                                            "decode_wall_ms": int((t_decode_end - t_decode_start) * 1000),
-                                            "decode_gpu_ms": int(decode_gpu_ms_this)
-                                            if decode_gpu_ms_this is not None else None,
-                                            "semantic_count": len(semantic_buffer),
-                                        },
-                                    )
+                                    logger.info(
+                                        "tts_stream_decode_batch",
+                                        extra={
+                                            "speaker": speaker,
+                                            "request_id": request_id,
+                                            "batch_index": decode_calls,
+                                            "decode_wall_ms": int((t_decode_end - t_decode_start) * 1000),
+                                            "decode_gpu_ms": int(decode_gpu_ms_this)
+                                            if decode_gpu_ms_this is not None else None,
+                                            "semantic_count": len(semantic_buffer),
+                                        },
+                                    )
*** End Patch


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/veenaModal/veena3modal/core/streaming_pipeline.py:
        for request_output in results_generator:
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)

            parse_start = time.time()
            # OPTIMIZATION: Only process NEW tokens (incremental parsing)


nl -ba veena3modal/core/streaming_pipeline.py | sed -n '420,620p'


Output:
Chunk ID: 08853d
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 3072
Output:
   420	                        decode_ms = (t_decode_end - t_decode_start) * 1000
   421	                        decode_wall_ms += decode_ms
   422	                        decode_wall_samples.append(decode_ms)
   423	                        if emit_timing and torch.cuda.is_available():
   424	                            decode_end_event.record()
   425	                            torch.cuda.synchronize()
   426	                            decode_gpu_ms_this = decode_start_event.elapsed_time(decode_end_event)
   427	                            decode_gpu_ms += decode_gpu_ms_this
   428	                            decode_gpu_samples.append(decode_gpu_ms_this)
   429	                            batch_gpu_ms.append(decode_gpu_ms_this)
   430
   431	                        if audio_bytes:
   432	                            total_samples_decoded = len(audio_bytes) // 2
   433
   434	                            if len(semantic_buffer) <= WINDOW_SIZE:
   435	                                # Early phase: extract new samples from full decode
   436	                                if total_samples_decoded > total_samples_emitted_to_user:
   437	                                    new_bytes_start = total_samples_emitted_to_user * 2
   438	                                    new_audio_bytes = audio_bytes[new_bytes_start:]
   439	                                else:
   440	                                    new_audio_bytes = b""
   441	                            else:
   442	                                # Windowed phase: take only the tail portion of the window decode
   443	                                # The last DECODE_INTERVAL tokens' worth of audio is the new content
   444	                                # Each token produces ~320 samples at 16kHz/50TPS
   445	                                new_samples_approx = DECODE_INTERVAL * 320
   446	                                if total_samples_decoded > new_samples_approx:
   447	                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
   448	                                else:
   449	                                    new_bytes_start = 0
   450	                                new_audio_bytes = audio_bytes[new_bytes_start:]
   451
   452	                            if new_audio_bytes:
   453	                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   454	                                    previous_chunk_tail,
   455	                                    new_audio_bytes,
   456	                                    sample_rate_hz=16000,
   457	                                    crossfade_ms=CROSSFADE_MS,
   458	                                )
   459
   460	                        samples_emitted_in_this_chunk = len(to_emit) // 2
   461	                        total_samples_emitted_to_user += samples_emitted_in_this_chunk
   462
   463	                                if to_emit:
   464	                                    total_audio_chunks += 1
   465	                                    if t_first_chunk_decoded is None:
   466	                                        t_first_chunk_decoded = t_decode_end
   467
   468	                                    if emit_timing:
   469	                                    logger.info(
   470	                                        "tts_stream_decode_batch",
   471	                                        extra={
   472	                                            "speaker": speaker,
   473	                                            "request_id": request_id,
   474	                                            "batch_index": decode_calls,
   475	                                            "decode_wall_ms": int((t_decode_end - t_decode_start) * 1000),
   476	                                            "decode_gpu_ms": int(decode_gpu_ms_this)
   477	                                            if decode_gpu_ms_this is not None else None,
   478	                                            "semantic_count": len(semantic_buffer),
   479	                                        },
   480	                                    )
   481
   482	                                    yield to_emit
   483
   484	        # Final chunk: decode any remaining semantic tokens
   485	        # Use the same 32 global tokens (they get pooled)
   486	        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   487	            audio_bytes = self.snac_decoder.decode_streaming(
   488	                semantic_ids=semantic_buffer,  # All semantic tokens
   489	                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],  # Use first 32 global tokens
   490	                use_sliding_window=False,  # Get all samples
   491	                trim_warmup=False
   492	            )
   493	            if audio_bytes:
   494	                # Only yield NEW samples
   495	                total_samples_decoded = len(audio_bytes) // 2
   496
   497	                if total_samples_decoded > total_samples_emitted_to_user:
   498	                    new_bytes_start = total_samples_emitted_to_user * 2
   499	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   500
   501	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   502	                        previous_chunk_tail,
   503	                        new_audio_bytes,
   504	                        sample_rate_hz=16000,
   505	                        crossfade_ms=CROSSFADE_MS,
   506	                    )
   507
   508	                    samples_emitted_in_final = len(to_emit) // 2
   509	                    total_samples_emitted_to_user += samples_emitted_in_final
   510
   511	                    if to_emit:
   512	                        total_audio_chunks += 1
   513	                        yield to_emit
   514
   515	        # Flush any remaining tail that was held back for crossfade
   516	        if previous_chunk_tail:
   517	            total_audio_chunks += 1
   518	            tail_samples = len(previous_chunk_tail) // 2
   519	            total_samples_emitted_to_user += tail_samples
   520	            yield previous_chunk_tail
   521
   522	        # Log completion summary
   523	        t_end = time.time()
   524	        audio_duration_s = total_samples_emitted_to_user / 16000  # 16kHz sample rate
   525	        ttfb_ms = (
   526	            (t_first_chunk_decoded - t_start) * 1000
   527	            if t_first_chunk_decoded is not None
   528	            else (t_first_token - t_start) * 1000
   529	            if t_first_token is not None
   530	            else 0
   531	        )
   532	        total_wall_ms = (t_end - t_start) * 1000
   533	        request_metrics.update(
   534	            {
   535	                "llm_request_id": request_id,
   536	                "generated_tokens": len(semantic_buffer) + len(global_buffer),
   537	                "semantic_tokens": len(semantic_buffer),
   538	                "global_tokens": len(global_buffer),
   539	                "llm_batch_count": len(batch_wall_ms),
   540	                "llm_batch_tokens": batch_token_counts,
   541	                "llm_batch_wall_ms_total": int(sum(batch_wall_ms)),
   542	                "llm_batch_wall_ms_min": int(min(batch_wall_ms)) if batch_wall_ms else 0,
   543	                "llm_batch_wall_ms_max": int(max(batch_wall_ms)) if batch_wall_ms else 0,
   544	                "llm_batch_wall_ms_p50": int(statistics.median(batch_wall_ms)) if batch_wall_ms else 0,
   545	                "llm_batch_gpu_ms_total": int(sum(batch_gpu_ms)),
   546	                "llm_batch_gpu_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
   547	                "llm_batch_gpu_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
   548	                "llm_batch_gpu_ms_p50": int(statistics.median(batch_gpu_ms)) if batch_gpu_ms else 0,
   549	                "llm_parse_ms": int(sum(parse_wall_ms_list)),
   550	                "llm_parse_ms_avg": int(sum(parse_wall_ms_list) / len(parse_wall_ms_list)) if parse_wall_ms_list else 0,
   551	                "decode_calls": decode_calls,
   552	                "llm_decode_wall_ms_total": int(decode_wall_ms),
   553	                "llm_decode_wall_ms_min": int(min(decode_wall_samples)) if decode_wall_samples else 0,
   554	                "llm_decode_wall_ms_max": int(max(decode_wall_samples)) if decode_wall_samples else 0,
   555	                "llm_decode_wall_ms_p50": int(statistics.median(decode_wall_samples)) if decode_wall_samples else 0,
   556	                "llm_decode_gpu_ms_total": int(decode_gpu_ms),
   557	                "llm_decode_gpu_ms_min": int(min(decode_gpu_samples)) if decode_gpu_samples else 0,
   558	                "llm_decode_gpu_ms_max": int(max(decode_gpu_samples)) if decode_gpu_samples else 0,
   559	                "ttfb_ms": int(ttfb_ms),
   560	                "generation_ms": int(total_wall_ms),
   561	                "audio_duration_seconds": audio_duration_s,
   562	                "audio_chunks": total_audio_chunks,
   563	                "samples_emitted": total_samples_emitted_to_user,
   564	                "text_chunked": False,
   565	                "concurrency_mode": "single",
   566	            }
   567	        )
   568	        if emit_timing:
   569	            logger.info("tts_stream_indic_complete", extra=request_metrics)
   570
   571	    async def generate_speech_stream_indic_first_chunk(
   572	        self,
   573	        speaker: str,
   574	        text: str,
   575	        temperature: float = DEFAULT_TEMPERATURE,
   576	        top_k: int = DEFAULT_TOP_K,
   577	        top_p: float = DEFAULT_TOP_P,
   578	        max_tokens: int = DEFAULT_MAX_TOKENS,
   579	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   580	        seed: Optional[int] = None,
   581	        request_id: Optional[str] = None,
   582	        request_metrics: Optional[Dict[str, Any]] = None,
   583	    ) -> AsyncGenerator[tuple, None]:
   584	        """
   585	        Generate speech for the FIRST chunk of multi-chunk text, capturing global tokens.
   586
   587	        This method is specifically for chunked generation: it yields (audio_bytes, global_ids)
   588	        tuples instead of just audio_bytes. The caller captures global_ids from the first
   589	        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.
   590
   591	        Use Case (chunked streaming with voice consistency):
   592	            globals_captured = None
   593	            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
   594	                if globals_captured is None and global_ids:
   595	                    globals_captured = global_ids  # Capture once
   596	                yield audio_bytes
   597	            # Now use globals_captured for subsequent chunks
   598
   599	        Args:
   600	            speaker: Speaker name
   601	            text: First text chunk to synthesize
   602	            temperature: Sampling temperature
   603	            top_k: Top-k sampling
   604	            top_p: Nucleus sampling
   605	            max_tokens: Max tokens to generate
   606	            repetition_penalty: Prevent repetition
   607	            seed: Random seed for reproducibility
   608
   609	        Yields:
   610	            Tuple of (audio_bytes, global_ids)
   611	            - audio_bytes: Raw PCM audio (int16, 16kHz)
   612	            - global_ids: List of 32 global token IDs (populated after first decode, else empty)
   613
   614	        Thread Safety:
   615	            This method is stateless and thread-safe. Each call creates its own
   616	            request-scoped state. No global state is modified or shared.
   617	        """
   618	        import time
   619	        t_start = time.time()
   620	        request_metrics = request_metrics or {}


nl -ba veena3modal/core/streaming_pipeline.py | sed -n '320,430p'


Output:
Chunk ID: 6dfbdd
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 1641
Output:
   320	        batch_token_counts: List[int] = []
   321	        batch_gpu_ms: List[float] = []
   322	        parse_wall_ms_list: List[float] = []
   323	        decode_calls = 0
   324	        decode_wall_ms = 0.0
   325	        decode_gpu_ms = 0.0
   326	        decode_wall_samples: List[float] = []
   327	        decode_gpu_samples: List[float] = []
   328
   329	        results_generator = self.model.engine.generate(
   330	            prompt=prompt,
   331	            sampling_params=sampling_params,
   332	            request_id=request_id,
   333	        )
   334
   335	        # Track timing
   336	        t_first_token = None
   337	        t_first_chunk_ready = None
   338	        t_first_chunk_decoded = None
   339	        iteration_count = 0
   340	        last_decode_count = 0
   341
   342	        # Track audio samples EMITTED to the user (not just decoded)
   343	        # BiCodec produces ~320 samples per semantic token at 16kHz
   344	        # CRITICAL: This tracks samples that have been YIELDED, not held in tail
   345	        total_samples_emitted_to_user = 0
   346	        # Keep a tail to enable crossfading between consecutive emitted chunks
   347	        previous_chunk_tail: Optional[bytes] = None
   348
   349	        # Stream tokens with BiCodec extraction
   350	        async for request_output in results_generator:
   351	            iteration_count += 1
   352	            t_iter = time.time()
   353
   354	            # Track first token received
   355	            if t_first_token is None:
   356	                t_first_token = t_iter
   357
   358	            # Extract generated token IDs
   359	            generated_ids = request_output.outputs[0].token_ids
   360
   361	            parse_start = time.time()
   362	            # OPTIMIZATION: Only process NEW tokens (incremental parsing)
   363	            # Old code: tokenizer.decode(ALL) + re.findall(ALL) = O(n²) per stream
   364	            # New code: parse only new tokens = O(1) per token
   365	            new_token_ids = generated_ids[processed_token_count:]
   366	            processed_token_count = len(generated_ids)
   367
   368	            # Parse new tokens and append to buffers (in-place modification)
   369	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
   370	            parse_wall_ms = (time.time() - parse_start) * 1000
   371	            parse_wall_ms_list.append(parse_wall_ms)
   372	            batch_token_counts.append(len(new_token_ids))
   373	            batch_wall_ms.append((time.time() - t_iter) * 1000 if t_iter else 0.0)
   374
   375	            # TRUE STREAMING with BiCodec two-phase generation:
   376	            # Phase 1: Generate 32 global tokens (pre-roll)
   377	            # Phase 2: Stream semantic tokens incrementally
   378
   379	            # Check if we have the 32 global tokens (pre-roll complete)
   380	            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   381	                # Pre-roll complete! Now we can stream semantic tokens
   382	                if t_first_chunk_ready is None and len(semantic_buffer) >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   383	                    t_first_chunk_ready = time.time()
   384
   385	                # Stream semantic tokens incrementally
   386	                # Only decode if we have enough NEW semantic tokens since last decode
   387	                semantic_count = len(semantic_buffer)
   388	                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   389	                    # Check if we should decode (every DECODE_INTERVAL new semantic tokens)
   390	                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
   391	                        last_decode_count = semantic_count
   392
   393	                        # OPTIMIZATION: Windowed decode instead of full re-decode
   394	                        # Only decode the last WINDOW_SIZE tokens (O(1) per chunk instead of O(n))
   395	                        # For first few chunks when total tokens < WINDOW_SIZE, decode all
   396	                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]
   397
   398	                        if len(semantic_buffer) <= WINDOW_SIZE:
   399	                            # Early phase: decode all tokens (window not yet relevant)
   400	                            decode_semantic = semantic_buffer
   401	                        else:
   402	                            # Windowed: only decode last WINDOW_SIZE tokens
   403	                            decode_semantic = semantic_buffer[-WINDOW_SIZE:]
   404
   405	                        decode_gpu_ms_this = None
   406	                        t_decode_start = time.time()
   407	                        if emit_timing and torch.cuda.is_available():
   408	                            torch.cuda.synchronize()
   409	                            decode_start_event = torch.cuda.Event(enable_timing=True)
   410	                            decode_end_event = torch.cuda.Event(enable_timing=True)
   411	                            decode_start_event.record()
   412	                        audio_bytes = await self.snac_decoder.decode_single_async(
   413	                            semantic_ids=decode_semantic,
   414	                            global_ids=window_global,
   415	                            trim_warmup=False,
   416	                            use_sliding_window=False,
   417	                        )
   418	                        t_decode_end = time.time()
   419	                        decode_calls += 1
   420	                        decode_ms = (t_decode_end - t_decode_start) * 1000
   421	                        decode_wall_ms += decode_ms
   422	                        decode_wall_samples.append(decode_ms)
   423	                        if emit_timing and torch.cuda.is_available():
   424	                            decode_end_event.record()
   425	                            torch.cuda.synchronize()
   426	                            decode_gpu_ms_this = decode_start_event.elapsed_time(decode_end_event)
   427	                            decode_gpu_ms += decode_gpu_ms_this
   428	                            decode_gpu_samples.append(decode_gpu_ms_this)
   429	                            batch_gpu_ms.append(decode_gpu_ms_this)
   430


nl -ba veena3modal/core/streaming_pipeline.py | sed -n '740,1060p'


Output:
Chunk ID: 75ff52
Wall time: 0.0517 seconds
Process exited with code 0
Original token count: 3343
Output:
   740	                use_sliding_window=False,
   741	                trim_warmup=False
   742	            )
   743	            if audio_bytes:
   744	                total_samples_decoded = len(audio_bytes) // 2
   745
   746	                if total_samples_decoded > total_samples_emitted_to_user:
   747	                    new_bytes_start = total_samples_emitted_to_user * 2
   748	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   749
   750	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   751	                        previous_chunk_tail,
   752	                        new_audio_bytes,
   753	                        sample_rate_hz=16000,
   754	                        crossfade_ms=CROSSFADE_MS,
   755	                    )
   756
   757	                    samples_emitted_in_final = len(to_emit) // 2
   758	                    total_samples_emitted_to_user += samples_emitted_in_final
   759
   760	                    if to_emit:
   761	                        total_audio_chunks += 1
   762	                        yield (to_emit, captured_globals)
   763
   764	        # Flush remaining tail
   765	        if previous_chunk_tail:
   766	            total_audio_chunks += 1
   767	            tail_samples = len(previous_chunk_tail) // 2
   768	            total_samples_emitted_to_user += tail_samples
   769	            yield (previous_chunk_tail, captured_globals)
   770
   771	        t_end = time.time()
   772	        audio_duration_s = total_samples_emitted_to_user / 16000
   773	        request_metrics.update(
   774	            {
   775	                "request_id": request_id,
   776	                "speaker": speaker,
   777	                "audio_duration_seconds": audio_duration_s,
   778	                "audio_chunks": total_audio_chunks,
   779	                "text_chunked": True,
   780	                "captured_global_count": len(captured_globals),
   781	                "semantic_tokens": len(semantic_buffer),
   782	                "generation_ms": int((t_end - t_start) * 1000),
   783	            }
   784	        )
   785	        logger.info("tts_stream_first_chunk_complete", extra=request_metrics)
   786
   787	    async def generate_speech_stream_indic_continuation(
   788	        self,
   789	        speaker: str,
   790	        text: str,
   791	        global_ids: List[int],
   792	        temperature: float = DEFAULT_TEMPERATURE,
   793	        top_k: int = DEFAULT_TOP_K,
   794	        top_p: float = DEFAULT_TOP_P,
   795	        max_tokens: int = DEFAULT_MAX_TOKENS,
   796	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   797	        seed: Optional[int] = None,
   798	        request_id: Optional[str] = None,
   799	        request_metrics: Optional[Dict[str, Any]] = None,
   800	    ) -> AsyncGenerator[bytes, None]:
   801	        """
   802	        Generate speech for CONTINUATION chunks using pre-captured global tokens.
   803
   804	        This is the key method for voice consistency in chunked generation:
   805	        - Uses global_ids from the first chunk to maintain identical voice
   806	        - Model generates only semantic tokens (skips global token generation)
   807	        - Ensures no voice drift across chunks
   808
   809	        CRITICAL for production:
   810	        - global_ids is request-scoped, passed explicitly (no shared state)
   811	        - Thread-safe: each request has its own state
   812	        - No global caching that could cause cross-request contamination
   813
   814	        Args:
   815	            speaker: Speaker name (must match first chunk)
   816	            text: Continuation text chunk to synthesize
   817	            global_ids: 32 global token IDs captured from first chunk
   818	            temperature: Sampling temperature
   819	            top_k: Top-k sampling
   820	            top_p: Nucleus sampling
   821	            max_tokens: Max tokens to generate
   822	            repetition_penalty: Prevent repetition
   823	            seed: Random seed (use same as first chunk for consistency)
   824
   825	        Yields:
   826	            Audio bytes (int16 PCM, 16kHz mono)
   827
   828	        Raises:
   829	            ValueError: If global_ids doesn't contain exactly 32 tokens
   830
   831	        Thread Safety:
   832	            Fully thread-safe. All state is request-scoped and passed explicitly.
   833	        """
   834	        import time
   835	        t_start = time.time()
   836	        request_metrics = request_metrics or {}
   837	        request_metrics.setdefault("request_id", request_id)
   838
   839	        # Validate global tokens
   840	        EXPECTED_GLOBAL_COUNT = 32
   841	        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
   842	            raise ValueError(
   843	                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
   844	                f"global_ids must be captured from first chunk generation."
   845	            )
   846
   847	        # Build prompt WITH pre-filled global tokens
   848	        # This tells the model to skip global generation and go straight to semantic tokens
   849	        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)
   850
   851	        # Configure sampling
   852	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
   853	        sampling_params = SamplingParams(
   854	            temperature=temperature,
   855	            top_k=top_k,
   856	            top_p=top_p,
   857	            max_tokens=max_tokens,
   858	            min_tokens=DEFAULT_MIN_TOKENS,
   859	            repetition_penalty=repetition_penalty,
   860	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
   861	            skip_special_tokens=False,
   862	            seed=seed,
   863	        )
   864
   865	        # BiCodec token buffers
   866	        semantic_buffer = []
   867	        global_buffer_unused = []  # Not used in continuation, but needed for parser API
   868	        processed_token_count = 0  # Track how many vLLM tokens we've parsed
   869	        total_audio_chunks = 0
   870
   871	        # BiCodec streaming configuration
   872	        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
   873	        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
   874	        CROSSFADE_MS = 50
   875
   876	        # Generate unique request ID
   877	        import uuid
   878	        if request_id is None:
   879	            request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
   880	            request_metrics["request_id"] = request_id
   881
   882	        # Reuse pre-warmed token parser (singleton, created once in __init__)
   883	        token_parser = self.token_parser
   884	        if token_parser is None:
   885	            raise RuntimeError("BiCodecTokenParser unavailable for continuation streaming")
   886
   887	        results_generator = self.model.engine.generate(
   888	            prompt=prompt,
   889	            sampling_params=sampling_params,
   890	            request_id=request_id,
   891	        )
   892
   893	        # Track state
   894	        last_decode_count = 0
   895	        total_samples_emitted_to_user = 0
   896	        previous_chunk_tail: Optional[bytes] = None
   897
   898	        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
   899	        async for request_output in results_generator:
   900	            generated_ids = request_output.outputs[0].token_ids
   901
   902	            # OPTIMIZATION: Incremental parsing (O(1) per new token)
   903	            # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
   904	            new_token_ids = generated_ids[processed_token_count:]
   905	            processed_token_count = len(generated_ids)
   906	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)
   907
   908	            semantic_count = len(semantic_buffer)
   909	            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   910	                if semantic_count - last_decode_count >= DECODE_INTERVAL:
   911	                    last_decode_count = semantic_count
   912
   913	                    # Use pre-captured global tokens for decoding
   914	                    if self.snac_decoder.enable_batching:
   915	                        audio_bytes = await self.snac_decoder.decode_single_async(
   916	                            semantic_ids=semantic_buffer,
   917	                            global_ids=global_ids,  # Use captured globals
   918	                            trim_warmup=False,
   919	                            use_sliding_window=False
   920	                        )
   921	                    else:
   922	                        audio_bytes = self.snac_decoder.decode_streaming(
   923	                            semantic_ids=semantic_buffer,
   924	                            global_ids=global_ids,  # Use captured globals
   925	                            use_sliding_window=False,
   926	                            trim_warmup=False
   927	                        )
   928
   929	                    if audio_bytes:
   930	                        total_samples_decoded = len(audio_bytes) // 2
   931
   932	                        if total_samples_decoded > total_samples_emitted_to_user:
   933	                            new_bytes_start = total_samples_emitted_to_user * 2
   934	                            new_audio_bytes = audio_bytes[new_bytes_start:]
   935
   936	                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
   937	                                previous_chunk_tail,
   938	                                new_audio_bytes,
   939	                                sample_rate_hz=16000,
   940	                                crossfade_ms=CROSSFADE_MS,
   941	                            )
   942
   943	                            samples_emitted_in_this_chunk = len(to_emit) // 2
   944	                            total_samples_emitted_to_user += samples_emitted_in_this_chunk
   945
   946	                            if to_emit:
   947	                                total_audio_chunks += 1
   948	                                yield to_emit
   949
   950	        # Final chunk processing
   951	        if len(semantic_buffer) > last_decode_count:
   952	            audio_bytes = self.snac_decoder.decode_streaming(
   953	                semantic_ids=semantic_buffer,
   954	                global_ids=global_ids,  # Use captured globals
   955	                use_sliding_window=False,
   956	                trim_warmup=False
   957	            )
   958	            if audio_bytes:
   959	                total_samples_decoded = len(audio_bytes) // 2
   960
   961	                if total_samples_decoded > total_samples_emitted_to_user:
   962	                    new_bytes_start = total_samples_emitted_to_user * 2
   963	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   964
   965	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   966	                        previous_chunk_tail,
   967	                        new_audio_bytes,
   968	                        sample_rate_hz=16000,
   969	                        crossfade_ms=CROSSFADE_MS,
   970	                    )
   971
   972	                    samples_emitted_in_final = len(to_emit) // 2
   973	                    total_samples_emitted_to_user += samples_emitted_in_final
   974
   975	                        if to_emit:
   976	                            total_audio_chunks += 1
   977	                            yield to_emit
   978
   979	        # Flush remaining tail
   980	        if previous_chunk_tail:
   981	            total_audio_chunks += 1
   982	            tail_samples = len(previous_chunk_tail) // 2
   983	            total_samples_emitted_to_user += tail_samples
   984	            yield previous_chunk_tail
   985
   986	        t_end = time.time()
   987	        request_metrics.update(
   988	            {
   989	                "request_id": request_id,
   990	                "speaker": speaker,
   991	                "audio_duration_seconds": total_samples_emitted_to_user / 16000,
   992	                "audio_chunks": total_audio_chunks,
   993	                "semantic_tokens": len(semantic_buffer),
   994	                "global_tokens": len(global_ids),
   995	                "generation_ms": int((t_end - t_start) * 1000),
   996	            }
   997	        )
   998	        logger.info("tts_stream_continuation_complete", extra=request_metrics)