Output:
Chunk ID: c57b24
Wall time: 0.0506 seconds
Process exited with code 0
Original token count: 25262
Output:
Total output lines: 1076

veena3modal/api/fastapi_app.py:5:Veena3 inference/streaming code from the Django project.
veena3modal/api/fastapi_app.py:35:    record_ttfb,
veena3modal/api/fastapi_app.py:38:    record_chunks_sent,
veena3modal/api/fastapi_app.py:176:        "x-llm-text-chunked": "text_chunked",
veena3modal/api/fastapi_app.py:177:        "x-llm-chunks-processed": "chunks_processed",
veena3modal/api/fastapi_app.py:281:        description="High-quality multilingual Text-to-Speech API with true streaming support.",
veena3modal/api/fastapi_app.py:355:        Supports streaming (M4) and non-streaming (M3) modes.
veena3modal/api/fastapi_app.py:356:        Currently implements non-streaming WAV generation only.
veena3modal/api/fastapi_app.py:476:            stream=req.stream,
veena3modal/api/fastapi_app.py:479:        metrics_request_received(speaker=speaker, stream=req.stream, format=req.format)
veena3modal/api/fastapi_app.py:482:        if req.stream:
veena3modal/api/fastapi_app.py:483:            return await _handle_streaming_request(req, request_id, text, speaker, start_time)
veena3modal/api/fastapi_app.py:485:        # Format encoding will be applied after generation (non-streaming only)
veena3modal/api/fastapi_app.py:486:        # For streaming, only WAV is supported
veena3modal/api/fastapi_app.py:495:                # Generate speech (non-streaming)
veena3modal/api/fastapi_app.py:496:                if req.chunking:
veena3modal/api/fastapi_app.py:497:                    audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
veena3modal/api/fastapi_app.py:565:                    "X-TTFB-ms": str(metrics.get("ttfb_ms", 0)),
veena3modal/api/fastapi_app.py:567:                    "X-Text-Chunked": str(metrics.get("text_chunked", False)).lower(),
veena3modal/api/fastapi_app.py:639:                    stream=False,
veena3modal/api/fastapi_app.py:647:                    text_chunked=metrics.get("text_chunked", False),
veena3modal/api/fastapi_app.py:648:                    ttfb_ms=metrics.get("ttfb_ms"),
veena3modal/api/fastapi_app.py:663:                    chunks_sent=1,
veena3modal/api/fastapi_app.py:669:                    stream=False,
veena3modal/api/fastapi_app.py:671:                record_ttfb(ttfb_seconds=metrics.get("ttfb_ms", 0) / 1000, speaker=speaker, stream=False)
veena3modal/api/fastapi_app.py:740:    async def _handle_streaming_request(req, request_id: str, text: str, speaker: str, start_time: float):
veena3modal/api/fastapi_app.py:742:        Handle streaming TTS request.
veena3modal/api/fastapi_app.py:744:        Returns a StreamingResponse that yields WAV header + PCM chunks.
veena3modal/api/fastapi_app.py:745:        True streaming: first bytes sent ASAP before full audio is generated.
veena3modal/api/fastapi_app.py:751:        # Check if streaming pipeline is available
veena3modal/api/fastapi_app.py:753:        if runtime is None or runtime.streaming_pipeline is None:
veena3modal/api/fastapi_app.py:783:        # Create async generator for streaming response
veena3modal/api/fastapi_app.py:784:        async def audio_stream_generator():
veena3modal/api/fastapi_app.py:785:            """Yields audio chunks as they're generated."""
veena3modal/api/fastapi_app.py:787:            first_chunk_sent = False
veena3modal/api/fastapi_app.py:788:            chunks_count = 0
veena3modal/api/fastapi_app.py:790:                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
veena3modal/api/fastapi_app.py:799:                    enable_chunking=req.chunking,
veena3modal/api/fastapi_app.py:802:                    chunks_count += 1
veena3modal/api/fastapi_app.py:804:                    # Fire-and-forget sentence storage after first chunk (don't block TTFB)
veena3modal/api/fastapi_app.py:805:                    if not first_chunk_sent:
veena3modal/api/fastapi_app.py:806:                        first_chunk_sent = True
veena3modal/api/fastapi_app.py:807:                        ttfb_ms = metrics.get("ttfb_ms", 0)
veena3modal/api/fastapi_app.py:812:                            ttfb_ms=ttfb_ms,
veena3modal/api/fastapi_app.py:813:                            chunk_size_bytes=len(audio_chunk),
veena3modal/api/fastapi_app.py:815:                        record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)
veena3modal/api/fastapi_app.py:822:                            stream=True,
veena3modal/api/fastapi_app.py:830:                            text_chunked=metrics.get("text_chunked", False),
veena3modal/api/fastapi_app.py:831:                            ttfb_ms=ttfb_ms,
veena3modal/api/fastapi_app.py:832:                            # audio_duration_seconds not known yet for streaming
veena3modal/api/fastapi_app.py:835:                    yield audio_chunk
veena3modal/api/fastapi_app.py:848:                    chunks_sent=chunks_count,
veena3modal/api/fastapi_app.py:854:                    stream=True,
veena3modal/api/fastapi_app.py:858:                record_chunks_sent(chunks=chunks_count, speaker=speaker)
veena3modal/api/fastapi_app.py:862:                # Log streaming failure
veena3modal/api/fastapi_app.py:875:                # Can't return error once streaming started
veena3modal/api/fastapi_app.py:876:                # Best we can do is stop yielding
veena3modal/api/fastapi_app.py:880:        # Note: Some headers can't be set dynamically during streaming
veena3modal/api/fastapi_app.py:881:        # TTFB and RTF will be logged but not in headers for streaming
veena3modal/api/fastapi_app.py:890:            "X-Chunking-Enabled": str(req.chunking).lower(),
veena3modal/api/fastapi_app.py:891:            # Transfer-Encoding: chunked is set automatically by StreamingResponse
veena3modal/api/fastapi_app.py:898:            audio_stream_generator(),
veena3modal/services/tts_runtime.py:40:    streaming_pipeline: Any = None
veena3modal/services/tts_runtime.py:56:    # OPTIMIZATION: Pre-computed global tokens per speaker (eliminates ~110ms pre-roll in streaming)
veena3modal/services/tts_runtime.py:86:    enable_chunked_prefill: Optional[bool] = None,
veena3modal/services/tts_runtime.py:108:        enable_chunked_prefill: Optional vLLM chunked prefill toggle.
veena3modal/services/tts_runtime.py:158:        if enable_chunked_prefill is not None:
veena3modal/services/tts_runtime.py:159:            engine_kwargs["enable_chunked_prefill"] = bool(enable_chunked_prefill)
veena3modal/services/tts_runtime.py:213:        # Initialize streaming pipeline (for M4)
veena3modal/services/tts_runtime.py:214:        streaming_pipeline = None
veena3modal/services/tts_runtime.py:216:            from veena3modal.core.streaming_pipeline import Veena3SlidingWindowPipeline
veena3modal/services/tts_runtime.py:217:            logger.info("Initializing streaming pipeline...")
veena3modal/services/tts_runtime.py:219:            streaming_pipeline = Veena3SlidingWindowPipeline(
veena3modal/services/tts_runtime.py:230:            streaming_pipeline=streaming_pipeline,
veena3modal/services/tts_runtime.py:260:            streaming_pipeline=streaming_pipeline,
veena3modal/services/tts_runtime.py:308:    OPTIMIZATION: BiCodec streaming requires 32 "global tokens" before any audio can be emitted.
veena3modal/services/tts_runtime.py:311:    and cache them. During streaming, we inject cached globals via build_prefix_with_globals(),
veena3modal/services/tts_runtime.py:314:    This runs synchronously at startup (adds ~5-10s to cold start, saves ~110ms per streaming TTFB).
veena3modal/services/tts_runtime.py:318:    from vllm import SamplingParams
veena3modal/services/tts_runtime.py:329:    sampling_params = SamplingParams(
veena3modal/services/tts_runtime.py:344:        async for request_output in runtime.model.engine.generate(
veena3modal/services/tts_runtime.py:349:            final_output = request_output
veena3modal/services/tts_runtime.py:401:            streaming_pipeline=runtime.streaming_pipeline,
veena3modal/services/tts_runtime.py:418:    Generate speech audio (non-streaming).
veena3modal/services/tts_runtime.py:426:        max_tokens: Maximum tokens to generate
veena3modal/services/tts_runtime.py:444:        "ttfb_ms": 0,
veena3modal/services/tts_runtime.py:446:        "tokens_generated": 0,
veena3modal/services/tts_runtime.py:471:        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time
veena3modal/services/tts_runtime.py:537:    # Convert numpy to torch tensor for process_chunk
veena3modal/services/tts_runtime.py:540:    # Apply super-resolution using process_chunk (16kHz -> 48kHz)
veena3modal/services/tts_runtime.py:541:    # process_chunk expects [batch, samples] or [samples] and returns [batch, samples]
veena3modal/services/tts_runtime.py:542:    upsampled_tensor = sr_service.process_chunk(audio_tensor)
veena3modal/services/tts_runtime.py:564:        16,  # fmt chunk size
veena3modal/services/tts_runtime.py:578:async def generate_speech_chunked(
veena3modal/services/tts_runtime.py:591:    Generate speech with automatic text chunking for long inputs.
veena3modal/services/tts_runtime.py:610:        "ttfb_ms": 0,
veena3modal/services/tts_runtime.py:612:        "chunks_processed": 0,
veena3modal/services/tts_runtime.py:614:        "text_chunked": False,
veena3modal/services/tts_runtime.py:624:        # Check if chunking is needed
veena3modal/services/tts_runtime.py:625:        if long_processor.should_chunk(text):
veena3modal/services/tts_runtime.py:626:            metrics["text_chunked"] = True
veena3modal/services/tts_runtime.py:627:            audio_bytes = await long_processor.generate_with_chunking(
veena3modal/services/tts_runtime.py:655:        metrics["ttfb_ms"] = metrics["generation_ms"]
veena3modal/services/tts_runtime.py:663:                    logger.info(f"Applying super-resolution to chunked audio ({len(audio_bytes)} bytes)...")
veena3modal/services/tts_runtime.py:689:async def generate_speech_streaming(
veena3modal/services/tts_runtime.py:698:    enable_chunking: bool = True,
veena3modal/services/tts_runtime.py:701:    Generate speech audio with true streaming (yields chunks as they're generated).
veena3modal/services/tts_runtime.py:703:    This is the core streaming implementation for M4.
veena3modal/services/tts_runtime.py:704:    First yield includes WAV header + first PCM chunk.
veena3modal/services/tts_runtime.py:705:    Subsequent yields are raw PCM chunks.
veena3modal/services/tts_runtime.py:713:        max_tokens: Maximum tokens to generate
veena3modal/services/tts_runtime.py:716:        enable_chunking: Enable text chunking for long inputs with voice consistency
veena3modal/services/tts_runtime.py:720:        - First yield: WAV header (44 bytes) prepended to first PCM chunk
veena3modal/services/tts_runtime.py:721:        - Subsequent yields: Raw PCM chunks (int16, 16kHz)
veena3modal/services/tts_runtime.py:722:        - Final yield: metrics_dict has final timing info
veena3modal/services/tts_runtime.py:725:        RuntimeError: If runtime not initialized or streaming pipeline unavailable
veena3modal/services/tts_runtime.py:732:    if runtime.streaming_pipeline is None:
veena3modal/services/tts_runtime.py:740:        "ttfb_ms": 0,
veena3modal/services/tts_runtime.py:741:        "chunks_sent": 0,
veena3modal/services/tts_runtime.py:744:        "text_chunked": False,
veena3modal/services/tts_runtime.py:748:    first_chunk_time = None
veena3modal/services/tts_runtime.py:752:    # Check if we need text chunking
veena3modal/services/tts_runtime.py:754:    needs_chunking = enable_chunking and long_processor.should_chunk(text)
veena3modal/services/tts_runtime.py:756:    if needs_chunking:
veena3modal/services/tts_runtime.py:757:        # Chunked streaming with voice consistency (global token caching)
veena3modal/services/tts_runtime.py:758:        metrics["text_chunked"] = True
veena3modal/services/tts_runtime.py:759:        async for audio_chunk, chunk_metrics in _stream_chunked_text(
veena3modal/services/tts_runtime.py:761:            max_tokens, repetition_penalty, seed, sample_rate
veena3modal/services/tts_runtime.py:763:            # First chunk: prepend WAV header
veena3modal/services/tts_runtime.py:764:            if first_chunk_time is None:
veena3modal/services/tts_runtime.py:765:                first_chunk_time = time.time()
veena3modal/services/tts_runtime.py:766:                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
veena3modal/services/tts_runtime.py:767:                # Create streaming WAV header (size=0 for unknown length)
veena3modal/services/tts_runtime.py:769:                audio_chunk = wav_header + audio_chunk
veena3modal/services/tts_runtime.py:771:            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
veena3modal/services/tts_runtime.py:772:            metrics["chunks_sent"] += 1
veena3modal/services/tts_runtime.py:776:            yield audio_chunk, metrics
veena3modal/services/tts_runtime.py:778:        # Simple streaming (no text chunking)
veena3modal/services/tts_runtime.py:786:            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
veena3modal/services/tts_runtime.py:799:            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
veena3modal/services/tts_runtime.py:810:        async for audio_chunk in stream_gen:
veena3modal/services/tts_runtime.py:811:            # First chunk: prepend WAV header
veena3modal/services/tts_runtime.py:812:            if first_chunk_time is None:
veena3modal/services/tts_runtime.py:813:                first_chunk_time = time.time()
veena3modal/services/tts_runtime.py:814:                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
veena3modal/services/tts_runtime.py:816:                audio_chunk = wav_header + audio_chunk
veena3modal/services/tts_runtime.py:818:            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
veena3modal/services/tts_runtime.py:819:            metrics["chunks_sent"] += 1
veena3modal/services/tts_runtime.py:823:            yield audio_chunk, metrics
veena3modal/services/tts_runtime.py:826:async def _stream_chunked_text(
veena3modal/services/tts_runtime.py:840:    Internal helper: stream chunked text with voice consistency.
veena3modal/services/tts_runtime.py:842:    Uses global token caching from first chunk to maintain voice across chunks.
veena3modal/services/tts_runtime.py:846:    # Chunk text using the long text processor's chunker
veena3modal/services/tts_runtime.py:847:    chunks = long_processor.chunk_text(text)
veena3modal/services/tts_runtime.py:849:    if not chunks:
veena3modal/services/tts_runtime.py:853:    previous_chunk_tail: Optional[bytes] = None
veena3modal/services/tts_runtime.py:854:    chunk_metrics = {"chunks_processed": 0}
veena3modal/services/tts_runtime.py:856:    for i, chunk_text in enumerate(chunks):
veena3modal/services/tts_runtime.py:857:        chunk_metrics["chunks_processed"] = i + 1
veena3modal/services/tts_runtime.py:860:            # First chunk: capture global tokens for voice consistency
veena3modal/services/tts_runtime.py:861:            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
veena3modal/services/tts_runtime.py:863:                text=chunk_text,
veena3modal/services/tts_runtime.py:874:                # Crossfade with previous tail (inter-chunk stitching)
veena3modal/services/tts_runtime.py:875:                to_emit, previous_chunk_tail = crossfade_bytes_int16(
veena3modal/services/tts_runtime.py:876:                    previous_chunk_tail,
veena3modal/services/tts_runtime.py:883:                    yield to_emit, chunk_metrics
veena3modal/services/tts_runtime.py:885:            # Continuation chunks: use captured globals for voice consistency
veena3modal/services/tts_runtime.py:887:                logger.warning(f"No captured globals for chunk {i+1}, using regular streaming")
veena3modal/services/tts_runtime.py:888:                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
veena3modal/services/tts_runtime.py:890:                    text=chunk_text,
veena3modal/services/tts_runtime.py:898:                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
veena3modal/services/tts_runtime.py:899:                        previous_chunk_tail,
veena3modal/services/tts_runtime.py:905:                        yield to_emit, chunk_metrics
veena3modal/services/tts_runtime.py:908:                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
veena3modal/services/tts_runtime.py:910:                    text=chunk_text,
veena3modal/services/tts_runtime.py:919:                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
veena3modal/services/tts_runtime.py:920:                        previous_chunk_tail,
veena3modal/services/tts_runtime.py:926:                        yield to_emit, chunk_metrics
veena3modal/services/tts_runtime.py:929:    if previous_chunk_tail:
veena3modal/services/tts_runtime.py:930:        yield previous_chunk_tail, chunk_metrics
veena3modal/core/streaming_pipeline.py:4:Implements streaming for both SNAC and BiCodec tokens.
veena3modal/core/streaming_pipeline.py:10:4. Apply sliding window (every N token pairs)
veena3modal/core/streaming_pipeline.py:11:5. Decode and stream audio chunks
veena3modal/core/streaming_pipeline.py:16:- ~10x CPU reduction in streaming hot loop
veena3modal/core/streaming_pipeline.py:20:2. Apply sliding window
veena3modal/core/streaming_pipeline.py:21:3. Decode and stream
veena3modal/core/streaming_pipeline.py:27:from vllm import SamplingParams
veena3modal/core/streaming_pipeline.py:49:    Streaming TTS pipeline using sliding window approach.
veena3modal/core/streaming_pipeline.py:54:    - Creating natural continuity between chunks
veena3modal/core/streaming_pipeline.py:66:        Initialize sliding window streaming pipeline.
veena3modal/core/streaming_pipeline.py:78:        # Saves ~123ms per streaming TTFB by avoiding repeated 166K-entry vocab iteration
veena3modal/core/streaming_pipeline.py:84:        print(f"🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)")
veena3modal/core/streaming_pipeline.py:86:    async def generate_speech_stream(
veena3modal/core/streaming_pipeline.py:97:        Generate speech audio with sliding window streaming.
veena3modal/core/streaming_pipeline.py:99:        Yields audio chunks using overlapping windows for smooth playback.
veena3modal/core/streaming_pipeline.py:106:            max_tokens: Max SNAC tokens to generate
veena3modal/core/streaming_pipeline.py:112:        print(f"\n🌊 Sliding window streaming generation")
veena3modal/core/streaming_pipeline.py:121:        sampling_params = SamplingParams(
veena3modal/core/streaming_pipeline.py:131:        print(f"🎲 Sampling: temp={temperature}, top_p={top_p}, sliding_window=28 tokens")
veena3modal/core/streaming_pipeline.py:133:        # Token buffer - keeps ALL tokens (not chunked)
veena3modal/core/streaming_pipeline.py:135:        total_tokens_generated = 0
veena3modal/core/streaming_pipeline.py:136:        total_audio_chunks = 0
veena3modal/core/streaming_pipeline.py:138:        # Generate tokens with vLLM (streaming)
veena3modal/core/streaming_pipeline.py:141:        # Generate unique request ID for concurrent streaming support
veena3modal/core/streaming_pipeline.py:145:        results_generator = self.model.engine.generate(
veena3modal/core/streaming_pipeline.py:151:        # Stream tokens with sliding window
veena3modal/core/streaming_pipeline.py:152:        async for request_output in results_generator:
veena3modal/core/streaming_pipeline.py:154:            generated_ids = request_output.outputs[0].token_ids
veena3modal/core/streaming_pipeline.py:157:            new_tokens = generated_ids[total_tokens_generated:]
veena3modal/core/streaming_pipeline.py:158:            …15262 tokens truncated…: str = "lipakshi", stream: bool = False, timeout: int = 120) -> Tuple[bytes, Dict]:
scripts/comprehensive_validation.py:188:        "stream": stream
scripts/comprehensive_validation.py:215:    audio_bytes, headers = generate_tts(MULTILINGUAL_TEXT_LARGE, speaker="lipakshi", stream=False, timeout=180)
scripts/comprehensive_validation.py:272:def test_chunking_behavior():
scripts/comprehensive_validation.py:273:    """Test 2: Analyze chunking behavior with debug logs"""
scripts/comprehensive_validation.py:276:    # Test various text lengths to trigger chunking
scripts/comprehensive_validation.py:278:        ("Short text (no chunking)", "Hello world!", 100),
scripts/comprehensive_validation.py:289:        audio_bytes, headers = generate_tts(text, speaker="lipakshi", stream=False, timeout=120)
scripts/comprehensive_validation.py:325:        audio_bytes, headers = generate_tts(text, speaker=speaker, stream=False, timeout=60)
scripts/comprehensive_validation.py:340:def test_ttfb():
scripts/comprehensive_validation.py:346:    # Non-streaming TTFB
scripts/comprehensive_validation.py:347:    log_info("Testing non-streaming TTFB...")
scripts/comprehensive_validation.py:349:    audio_bytes, headers = generate_tts(test_text, speaker="lipakshi", stream=False, timeout=60)
scripts/comprehensive_validation.py:350:    ttfb_non_stream = time.time() - t_start
scripts/comprehensive_validation.py:353:        log_success(f"Non-streaming TTFB: {ttfb_non_stream*1000:.0f}ms")
scripts/comprehensive_validation.py:355:        log_error("Non-streaming generation failed")
scripts/comprehensive_validation.py:359:    log_info("Testing streaming TTFB...")
scripts/comprehensive_validation.py:362:        payload = {"text": test_text, "speaker": "lipakshi", "stream": True}
scripts/comprehensive_validation.py:365:        response = requests.post(url, json=payload, stream=True, timeout=60)
scripts/comprehensive_validation.py:367:        # Read first chunk
scripts/comprehensive_validation.py:368:        first_chunk = None
scripts/comprehensive_validation.py:369:        for chunk in response.iter_content(chunk_size=1024):
scripts/comprehensive_validation.py:370:            if chunk:
scripts/comprehensive_validation.py:371:                first_chunk = chunk
scripts/comprehensive_validation.py:374:        ttfb_stream = time.time() - t_start
scripts/comprehensive_validation.py:376:        if first_chunk:
scripts/comprehensive_validation.py:377:            log_success(f"Streaming TTFB: {ttfb_stream*1000:.0f}ms")
scripts/comprehensive_validation.py:385:    if ttfb_non_stream < 5.0:
scripts/comprehensive_validation.py:386:        log_success(f"Non-streaming TTFB meets target (<5s)")
scripts/comprehensive_validation.py:389:        log_warning(f"Non-streaming TTFB above target: {ttfb_non_stream:.2f}s")
scripts/comprehensive_validation.py:410:                stream=False,
scripts/comprehensive_validation.py:468:            audio_bytes, headers = generate_tts(text, speaker=speaker, stream=False, timeout=60)
scripts/comprehensive_validation.py:498:        ("Chunking Behavior", test_chunking_behavior),
scripts/comprehensive_validation.py:500:        ("TTFB Measurement", test_ttfb),
scripts/validate_chunking_asr.py:5:Verifies that all text chunks were processed and spoken correctly
scripts/validate_chunking_asr.py:9:1. No chunks were skipped
scripts/validate_chunking_asr.py:29:        "audio": "/tmp/chunking_tests/test1_short.wav",
scripts/validate_chunking_asr.py:31:        "expected_chunked": False
scripts/validate_chunking_asr.py:34:        "audio": "/tmp/chunking_tests/test2_medium.wav",
scripts/validate_chunking_asr.py:35:        "original": "This is a medium length text that should trigger chunking. It contains multiple sentences spanning several lines. Each sentence provides meaningful content that needs to be preserved. The chunking algorithm should split this at natural sentence boundaries. This ensures good prosody and natural-sounding speech. The final audio should be seamlessly stitched together. No pops or clicks should be audible at chunk boundaries. The crossfade algorithm handles smooth transitions. This is the final sentence of our medium test. We continue with more sentences to ensure we exceed the 800 character threshold that triggers chunking. Here we add even more content to make absolutely sure. The algorithm will intelligently split this text. Each chunk will be processed separately. Then the audio will be stitched with crossfade. The result should sound natural and continuous.",
scripts/validate_chunking_asr.py:36:        "expected_chunked": True
scripts/validate_chunking_asr.py:39:        "audio": "/tmp/chunking_tests/test3_long_story.wav",
scripts/validate_chunking_asr.py:41:        "expected_chunked": True
scripts/validate_chunking_asr.py:108:        print(f"Expected chunked: {test_data['expected_chunked']}")
scripts/validate_chunking_asr.py:154:            print(f"   This may indicate missing chunks or poor transcription")
scripts/validate_chunking_asr.py:191:    print("❌ Low similarity (<40%) = Possible missing chunks or errors")
scripts/stress_test_optimized_local.py:147:    ttfb_ms: float = 0.0
scripts/stress_test_optimized_local.py:148:    server_ttfb_ms: float = 0.0
scripts/stress_test_optimized_local.py:155:    stream: bool = False
scripts/stress_test_optimized_local.py:167:    chunking: bool
scripts/stress_test_optimized_local.py:219:    def ttfb_stats(self) -> Dict[str, float]:
scripts/stress_test_optimized_local.py:220:        vals = [r.server_ttfb_ms for r in self.successes if r.server_ttfb_ms >= 0]
scripts/stress_test_optimized_local.py:351:        "x-llm-text-chunked": "text_chunked",
scripts/stress_test_optimized_local.py:352:        "x-llm-chunks-processed": "chunks_processed",
scripts/stress_test_optimized_local.py:371:                    "text_chunked",
scripts/stress_test_optimized_local.py:403:    chunking: bool,
scripts/stress_test_optimized_local.py:405:    stream: bool = False,
scripts/stress_test_optimized_local.py:412:        "stream": stream,
scripts/stress_test_optimized_local.py:413:        "chunking": chunking,
scripts/stress_test_optimized_local.py:419:    ttfb_time = 0.0
scripts/stress_test_optimized_local.py:422:        if stream:
scripts/stress_test_optimized_local.py:423:            async with client.stream(
scripts/stress_test_optimized_local.py:432:                async for chunk in response.aiter_bytes():
scripts/stress_test_optimized_local.py:434:                        ttfb_time = (time.time() - start) * 1000
scripts/stress_test_optimized_local.py:436:                    total_bytes += len(chunk)
scripts/stress_test_optimized_local.py:448:                        ttfb_ms=ttfb_time,
scripts/stress_test_optimized_local.py:454:                        stream=True,
scripts/stress_test_optimized_local.py:457:                        server_ttfb_ms=_normalize_time_ms(response.headers.get("x-ttfb-ms")),
scripts/stress_test_optimized_local.py:464:                    ttfb_ms=ttfb_time,
scripts/stress_test_optimized_local.py:468:                    stream=True,
scripts/stress_test_optimized_local.py:483:            server_ttfb = _normalize_time_ms(response.headers.get("x-ttfb-ms"))
scripts/stress_test_optimized_local.py:499:                    ttfb_ms=latency,
scripts/stress_test_optimized_local.py:500:                    server_ttfb_ms=server_ttfb,
scripts/stress_test_optimized_local.py:506:                    stream=False,
scripts/stress_test_optimized_local.py:515:                ttfb_ms=latency,
scripts/stress_test_optimized_local.py:516:                server_ttfb_ms=server_ttfb,
scripts/stress_test_optimized_local.py:521:                stream=False,
scripts/stress_test_optimized_local.py:535:            stream=stream,
scripts/stress_test_optimized_local.py:545:            stream=stream,
scripts/stress_test_optimized_local.py:554:    chunking: bool,
scripts/stress_test_optimized_local.py:557:    stream: bool,
scripts/stress_test_optimized_local.py:560:    mode = "streaming" if stream else "non-streaming"
scripts/stress_test_optimized_local.py:567:        chunking=chunking,
scripts/stress_test_optimized_local.py:586:                chunking=chunking,
scripts/stress_test_optimized_local.py:588:                stream=stream,
scripts/stress_test_optimized_local.py:616:                    stream=stream,
scripts/stress_test_optimized_local.py:629:    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}, chunking={level.chunking}")
scripts/stress_test_optimized_local.py:649:    ttfb = level.ttfb_stats
scripts/stress_test_optimized_local.py:650:    if ttfb and not level.mode.startswith("stream"):
scripts/stress_test_optimized_local.py:653:            f"    min={ttfb['min']:.0f} avg={ttfb['avg']:.0f} "
scripts/stress_test_optimized_local.py:654:            f"p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}"
scripts/stress_test_optimized_local.py:657:    if level.mode == "non-streaming":
scripts/stress_test_optimized_local.py:763:        token_ms, token_count = level.timing_rate("generation_ms", "llm_token_total")
scripts/stress_test_optimized_local.py:821:            "chunking": level.chunking,
scripts/stress_test_optimized_local.py:827:            "ttfb_ms": level.ttfb_stats,
scripts/stress_test_optimized_local.py:859:                    "ttfb_ms": r.ttfb_ms,
scripts/stress_test_optimized_local.py:860:                    "server_ttfb_ms": r.server_ttfb_ms,
scripts/stress_test_optimized_local.py:893:    mode_label = "streaming" if args.stream else "non-streaming"
scripts/stress_test_optimized_local.py:896:    print(f"\nWarming up (1 request, non-streaming)...")
scripts/stress_test_optimized_local.py:902:        chunking=args.chunking,
scripts/stress_test_optimized_local.py:905:        stream=False,
scripts/stress_test_optimized_local.py:918:            chunking=args.chunking,
scripts/stress_test_optimized_local.py:921:            stream=args.stream,
scripts/stress_test_optimized_local.py:943:    parser.add_argument("--chunking", action="store_true", help="Enable server-side chunking", default=True)
scripts/stress_test_optimized_local.py:944:    parser.add_argument("--no-chunking", action="store_false", dest="chunking", help="Disable chunking")
scripts/stress_test_optimized_local.py:945:    parser.add_argument("--stream", action="store_true", help="Use streaming endpoint")
scripts/stress_test_local.py:12:- Real streaming TTFB measurement (byte-level timing)
scripts/stress_test_local.py:25:    python scripts/stress_test_local.py --stream-only
scripts/stress_test_local.py:27:    # Non-streaming only
scripts/stress_test_local.py:28:    python scripts/stress_test_local.py --no-stream
scripts/stress_test_local.py:165:    ttfb_ms: float = 0.0  # Time to first byte (from HTTP response start)
scripts/stress_test_local.py:166:    server_ttfb_ms: float = 0.0  # Server-reported TTFB (X-TTFB-ms header)
scripts/stress_test_local.py:172:    stream: bool = False
scripts/stress_test_local.py:182:    mode: str  # "non-streaming" or "streaming"
scripts/stress_test_local.py:226:    def ttfb_stats(self) -> Dict[str, float]:
scripts/stress_test_local.py:227:        ttfbs = [r.ttfb_ms for r in self.successes if r.ttfb_ms > 0]
scripts/stress_test_local.py:228:        if not ttfbs:
scripts/stress_test_local.py:231:            "min": min(ttfbs),
scripts/stress_test_local.py:232:            "avg": statistics.mean(ttfbs),
scripts/stress_test_local.py:233:            "p50": self._percentile(ttfbs, 50),
scripts/stress_test_local.py:234:            "p95": self._percentile(ttfbs, 95),
scripts/stress_test_local.py:235:            "max": max(ttfbs),
scripts/stress_test_local.py:263:    stream: bool = False,
scripts/stress_test_local.py:271:        "stream": stream,
scripts/stress_test_local.py:276:    ttfb_time = 0.0
scripts/stress_test_local.py:279:        if stream:
scripts/stress_test_local.py:281:            async with client.stream(
scripts/stress_test_local.py:289:                async for chunk in response.aiter_bytes():
scripts/stress_test_local.py:291:                        ttfb_time = (time.time() - start) * 1000
scripts/stress_test_local.py:293:                    total_bytes += len(chunk)
scripts/stress_test_local.py:306:                        ttfb_ms=ttfb_time,
scripts/stress_test_local.py:311:                        stream=True,
scripts/stress_test_local.py:318:                        ttfb_ms=ttfb_time,
scripts/stress_test_local.py:322:                        stream=True,
scripts/stress_test_local.py:325:            # Non-streaming: standard request
scripts/stress_test_local.py:335:            server_ttfb = 0.0
scripts/stress_test_local.py:336:            ttfb_header = response.headers.get("X-TTFB-ms")
scripts/stress_test_local.py:337:            if ttfb_header:
scripts/stress_test_local.py:339:                    server_ttfb = float(ttfb_header)
scripts/stress_test_local.py:360:                    ttfb_ms=latency,  # Non-streaming: TTFB = total latency
scripts/stress_test_local.py:361:                    server_ttfb_ms=server_ttfb,
scripts/stress_test_local.py:366:                    stream=False,
scripts/stress_test_local.py:377:                    stream=False,
scripts/stress_test_local.py:385:            speaker=speaker, text_length=len(text), stream=stream,
scripts/stress_test_local.py:392:            speaker=speaker, text_length=len(text), stream=stream,
scripts/stress_test_local.py:403:    stream: bool = False,
scripts/stress_test_local.py:407:    mode = "streaming" if stream else "non-streaming"
scripts/stress_test_local.py:430:            return await make_request(client, base_url, text, speaker, stream=stream)
scripts/stress_test_local.py:453:                error=str(r)[:200], stream=stream,
scripts/stress_test_local.py:469:    ttfb = level.ttfb_stats
scripts/stress_test_local.py:488:    if ttfb and level.mode == "streaming":
scripts/stress_test_local.py:490:        print(f"    min={ttfb['min']:.0f}  avg={ttfb['avg']:.0f}  "
scripts/stress_test_local.py:491:              f"p50={ttfb['p50']:.0f}  p95={ttfb['p95']:.0f}  max={ttfb['max']:.0f}")
scripts/stress_test_local.py:569:    if not args.stream_only:
scripts/stress_test_local.py:570:        test_modes.append(False)  # non-streaming
scripts/stress_test_local.py:571:    if not args.no_stream:
scripts/stress_test_local.py:572:        test_modes.append(True)  # streaming
scripts/stress_test_local.py:582:    warmup = await run_level(base_url, num_requests=2, concurrency=1, text_category="short", stream=False)
scripts/stress_test_local.py:590:    for stream in test_modes:
scripts/stress_test_local.py:591:        mode_label = "STREAMING" if stream else "NON-STREAMING"
scripts/stress_test_local.py:603:            print(f"\n>>> {conc} concurrent, {num_requests} total requests, text={text_cat}, stream={stream}")
scripts/stress_test_local.py:610:                stream=stream,
scripts/stress_test_local.py:648:            "ttfb_ms": level.ttfb_stats,
scripts/stress_test_local.py:668:    parser.add_argument("--stream-only", action="store_true", help="Only test streaming")
scripts/stress_test_local.py:669:    parser.add_argument("--no-stream", action="store_true", help="Only test non-streaming")
scripts/measure_ttfb_detailed.py:5:This script will test both streaming and non-streaming modes and show
scripts/measure_ttfb_detailed.py:17:def measure_streaming_ttfb(text="Hello world, this is a test of streaming audio generation.", speaker="lipakshi"):
scripts/measure_ttfb_detailed.py:19:    Measure TTFB for streaming mode.
scripts/measure_ttfb_detailed.py:29:        "stream": True,
scripts/measure_ttfb_detailed.py:42:        # Make request with stream=True
scripts/measure_ttfb_detailed.py:43:        response = requests.post(API_URL, json=payload, headers=headers, stream=True, timeout=30)
scripts/measure_ttfb_detailed.py:59:        # Read first chunk
scripts/measure_ttfb_detailed.py:60:        first_chunk = None
scripts/measure_ttfb_detailed.py:61:        chunk_count = 0
scripts/measure_ttfb_detailed.py:64:        for chunk in response.iter_content(chunk_size=4096):
scripts/measure_ttfb_detailed.py:65:            if chunk:
scripts/measure_ttfb_detailed.py:66:                chunk_count += 1
scripts/measure_ttfb_detailed.py:67:                total_bytes += len(chunk)
scripts/measure_ttfb_detailed.py:69:                if first_chunk is None:
scripts/measure_ttfb_detailed.py:70:                    first_chunk = chunk
scripts/measure_ttfb_detailed.py:71:                    t2_first_chunk = time.time() - t0_request_start
scripts/measure_ttfb_detailed.py:72:                    print(f"\n🎵 T2 = {t2_first_chunk:.3f}s - FIRST AUDIO CHUNK RECEIVED")
scripts/measure_ttfb_detailed.py:73:                    print(f"   └─ Size: {len(chunk)} bytes")
scripts/measure_ttfb_detailed.py:74:                    print(f"   └─ TTFB (True): {t2_first_chunk:.3f}s = {t2_first_chunk*1000:.0f}ms")
scripts/measure_ttfb_detailed.py:77:                    if chunk.startswith(b'RIFF'):
scripts/measure_ttfb_detailed.py:79:                        if len(chunk) > 44:
scripts/measure_ttfb_detailed.py:80:                            print(f"   └─ Audio data starts: {len(chunk)-44} bytes in first chunk")
scripts/measure_ttfb_detailed.py:86:        print(f"   └─ Total chunks: {chunk_count}")
scripts/measure_ttfb_detailed.py:93:        print(f"   ├─ First chunk received:  {t2_first_chunk*1000:>7.0f}ms  ← TRUE TTFB")
scripts/measure_ttfb_detailed.py:97:            'ttfb': t2_first_chunk,
scripts/measure_ttfb_detailed.py:99:            'chunks': chunk_count,
scripts/measure_ttfb_detailed.py:110:def measure_non_streaming_ttfb(text="Hello world, this is a test.", speaker="lipakshi"):
scripts/measure_ttfb_detailed.py:112:    Measure TTFB for non-streaming mode.
scripts/measure_ttfb_detailed.py:113:    Note: In non-streaming, we receive the entire audio at once.
scripts/measure_ttfb_detailed.py:122:        "stream": False,
scripts/measure_ttfb_detailed.py:147:        server_ttfb = None
scripts/measure_ttfb_detailed.py:152:                    server_ttfb = float(value) / 1000  # Convert to seconds
scripts/measure_ttfb_detailed.py:163:        if server_ttfb:
scripts/measure_ttfb_detailed.py:164:            print(f"   ├─ Server reported (gen):    {server_ttfb*1000:>7.0f}ms  ← Generation time")
scripts/measure_ttfb_detailed.py:165:            network_time = t1_response_complete - server_ttfb
scripts/measure_ttfb_detailed.py:168:        print(f"\n   Note: Non-streaming TTFB = full generation time")
scripts/measure_ttfb_detailed.py:173:            'server_ttfb': server_ttfb,
scripts/measure_ttfb_detailed.py:196:    # Test 1: Short text, streaming
scripts/measure_ttfb_detailed.py:198:    streaming_short = measure_streaming_ttfb(
scripts/measure_ttfb_detailed.py:205:    # Test 2: Short text, non-streaming
scripts/measure_ttfb_detailed.py:207:    non_streaming_short = measure_non_streaming_ttfb(
scripts/measure_ttfb_detailed.py:214:    # Test 3: Longer text, streaming
scripts/measure_ttfb_detailed.py:217:    streaming_long = measure_streaming_ttfb(
scripts/measure_ttfb_detailed.py:227:    if streaming_short:
scripts/measure_ttfb_detailed.py:228:        print(f"\n✅ Streaming (short):    TTFB = {streaming_short['ttfb']*1000:>6.0f}ms")
scripts/measure_ttfb_detailed.py:229:    if non_streaming_short:
scripts/measure_ttfb_detailed.py:230:        print(f"✅ Non-streaming (short): Time = {non_streaming_short['total_time']*1000:>6.0f}ms (full generation)")
scripts/measure_ttfb_detailed.py:231:    if streaming_long:
scripts/measure_ttfb_detailed.py:232:        print(f"✅ Streaming (long):     TTFB = {streaming_long['ttfb']*1000:>6.0f}ms")
scripts/measure_ttfb_detailed.py:235:    print(f"   • Streaming TTFB = Time to first audio chunk (TRUE TTFB)")
scripts/measure_ttfb_detailed.py:236:    print(f"   • Non-streaming 'TTFB' = Full generation time (all audio ready)")


Output:
Chunk ID: 18f798
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2426
Output:

    except Exception as e:
        logger.error(f"Chunked speech generation failed: {e}")
        raise

from typing import AsyncGenerator

async def generate_speech_streaming(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    enable_chunking: bool = True,
) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
    """
    Generate speech audio with true streaming (yields chunks as they're generated).

    This is the core streaming implementation for M4.
    First yield includes WAV header + first PCM chunk.
    Subsequent yields are raw PCM chunks.

    Args:
        text: Text to synthesize (already normalized)
        speaker: Internal speaker name (resolved)
        temperature: Sampling temperature
        top_k: Top-k sampling
        top_p: Nucleus sampling
        max_tokens: Maximum tokens to generate
        repetition_penalty: Repetition penalty
        seed: Random seed for reproducibility
        enable_chunking: Enable text chunking for long inputs with voice consistency

    Yields:
        Tuple of (audio_bytes, metrics_dict)
        - First yield: WAV header (44 bytes) prepended to first PCM chunk
        - Subsequent yields: Raw PCM chunks (int16, 16kHz)
        - Final yield: metrics_dict has final timing info

    Raises:
        RuntimeError: If runtime not initialized or streaming pipeline unavailable
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()

    if runtime.streaming_pipeline is None:
        raise RuntimeError("Streaming pipeline not available")

    # Import audio utils for WAV header
    from veena3modal.audio.utils import create_wav_header

    # Metrics tracking
    metrics = {
        "ttfb_ms": 0,
        "chunks_sent": 0,
        "total_bytes": 0,
        "audio_duration_seconds": 0.0,
        "text_chunked": False,
    }

    start_time = time.time()
    first_chunk_time = None
    total_pcm_bytes = 0
    sample_rate = 16000  # BiCodec sample rate

    # Check if we need text chunking
    long_processor = _get_long_text_processor(runtime)
    needs_chunking = enable_chunking and long_processor.should_chunk(text)

    if needs_chunking:
        # Chunked streaming with voice consistency (global token caching)
        metrics["text_chunked"] = True
        async for audio_chunk, chunk_metrics in _stream_chunked_text(
            runtime, long_processor, text, speaker, temperature, top_k, top_p,
            max_tokens, repetition_penalty, seed, sample_rate
        ):
            # First chunk: prepend WAV header
            if first_chunk_time is None:
                first_chunk_time = time.time()
                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                # Create streaming WAV header (size=0 for unknown length)
                wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
                audio_chunk = wav_header + audio_chunk

            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
            metrics["chunks_sent"] += 1
            metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
            metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)

            yield audio_chunk, metrics
    else:
        # Simple streaming (no text chunking)
        # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
        # use continuation mode to skip the ~110ms global token pre-roll.
        # The model jumps straight to semantic token generation.
        cached_globals = runtime.speaker_global_cache.get(speaker)

        if cached_globals:
            # Fast path: use cached globals, skip global token generation entirely
            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                speaker=speaker,
                text=text,
                global_ids=cached_globals,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
            )
        else:
            # Fallback: no cached globals, generate them inline (original path)
            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
                speaker=speaker,
                text=text,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
            )

        async for audio_chunk in stream_gen:
            # First chunk: prepend WAV header
            if first_chunk_time is None:
                first_chunk_time = time.time()
                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
                audio_chunk = wav_header + audio_chunk

            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
            metrics["chunks_sent"] += 1
            metrics["total_bytes"] = total_pcm_bytes + 44
            metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)

            yield audio_chunk, metrics

async def _stream_chunked_text(
    runtime: TTSRuntime,
    long_processor,
    text: str,
    speaker: str,
    temperature: float,
    top_k: int,
    top_p: float,
    max_tokens: int,
    repetition_penalty: float,
    seed: Optional[int],
    sample_rate: int,
) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
    """
    Internal helper: stream chunked text with voice consistency.

    Uses global token caching from first chunk to maintain voice across chunks.
    """
    from veena3modal.audio.crossfade import crossfade_bytes_int16

    # Chunk text using the long text processor's chunker
    chunks = long_processor.chunk_text(text)

    if not chunks:
        return

    captured_globals: Optional[List[int]] = None
    previous_chunk_tail: Optional[bytes] = None
    chunk_metrics = {"chunks_processed": 0}

    for i, chunk_text in enumerate(chunks):
        chunk_metrics["chunks_processed"] = i + 1

        if i == 0:
            # First chunk: capture global tokens for voice consistency
            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
                speaker=speaker,
                text=chunk_text,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
            ):
                if captured_globals is None and global_ids:
                    captured_globals = global_ids

                # Crossfade with previous tail (inter-chunk stitching)
                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                    previous_chunk_tail,
                    audio_bytes,
                    sample_rate_hz=sample_rate,
                    crossfade_ms=50,
                )

                if to_emit:
                    yield to_emit, chunk_metrics
        else:
            # Continuation chunks: use captured globals for voice consistency
            if captured_globals is None:
                logger.warning(f"No captured globals for chunk {i+1}, using regular streaming")
                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
                    speaker=speaker,
                    text=chunk_text,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                ):
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        audio_bytes,
                        sample_rate_hz=sample_rate,
                        crossfade_ms=50,
                    )
                    if to_emit:
                        yield to_emit, chunk_metrics
            else:
                # Use continuation method with cached globals
                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                    speaker=speaker,
                    text=chunk_text,
                    global_ids=captured_globals,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                ):
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        audio_bytes,
                        sample_rate_hz=sample_rate,
                        crossfade_ms=50,
                    )
                    if to_emit:
                        yield to_emit, chunk_metrics

    # Flush remaining tail
    if previous_chunk_tail:
        yield previous_chunk_tail, chunk_metrics


Output:
Chunk ID: 2a0991
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 3800
Output:
"""
Veena3 Streaming Pipeline - Sliding Window Approach

Implements streaming for both SNAC and BiCodec tokens.

For BiCodec (Spark TTS):
1. Generate tokens from vLLM
2. Parse NEW token IDs incrementally (O(1) per token via cache)
3. Buffer semantic + global tokens
4. Apply sliding window (every N token pairs)
5. Decode and stream audio chunks

OPTIMIZATION (Dec 2025):
- Replaced O(n²) pattern (decode-all + regex-all per iteration)
- Now uses incremental token parsing with BiCodecTokenParser
- ~10x CPU reduction in streaming hot loop

For SNAC (legacy):
1. Filter SNAC token IDs directly
2. Apply sliding window
3. Decode and stream
"""

import asyncio
import re
from typing import AsyncGenerator, Optional, List
from vllm import SamplingParams
from veena3modal.audio.crossfade import crossfade_bytes_int16
from veena3modal.core.token_utils import BiCodecTokenParser

from veena3modal.core.constants import (
    CODE_END_TOKEN_ID,
    CODE_START_TOKEN_ID,
    SNAC_MIN_ID,
    SNAC_MAX_ID,
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_MIN_TOKENS,
    DEFAULT_REPETITION_PENALTY,
    DEFAULT_SEED,
)

class Veena3SlidingWindowPipeline:
    """
    Streaming TTS pipeline using sliding window approach.

    This eliminates choppy audio and popping artifacts by:
    - Decoding overlapping 28-token windows (4 frames)
    - Keeping only the middle 2048 samples from each decode
    - Creating natural continuity between chunks

    Based on the official Canopy Labs implementation.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        snac_decoder,
    ):
        """
        Initialize sliding window streaming pipeline.

        Args:
            model: Veena3Model instance
            prompt_builder: Veena3PromptBuilder instance
            snac_decoder: SNACDecoder instance (with batching enabled)
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.snac_decoder = snac_decoder

        # OPTIMIZATION: Pre-warm BiCodecTokenParser once at init, not per-request
        # Saves ~123ms per streaming TTFB by avoiding repeated 166K-entry vocab iteration
        tokenizer = getattr(model, "tokenizer", None)
        if tokenizer is None:
            tokenizer = getattr(model.engine, "tokenizer", None)
        self.token_parser = BiCodecTokenParser(tokenizer) if tokenizer else None

        print(f"🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)")

    async def generate_speech_stream(
        self,
        description: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech audio with sliding window streaming.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            description: Character/voice description
            text: Text to synthesize (with optional <emotion> tags)
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops

        Yields:
            Audio bytes (int16 PCM, 24kHz mono)
        """
        print(f"\n🌊 Sliding window streaming generation")
        print(f"📝 Description: {description[:80]}...")
        print(f"💬 Text: {text}")

        # Build prompt
        prompt = self.prompt_builder.build_prefix(description, text)
        print(f"✅ Prompt built ({len(prompt)} chars)")

        # Configure sampling
        sampling_params = SamplingParams(
            temperature=temperature,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop_token_ids=[CODE_END_TOKEN_ID],  # Only audio EOS
            seed=seed,  # None = random, int = reproducible
        )

        print(f"🎲 Sampling: temp={temperature}, top_p={top_p}, sliding_window=28 tokens")

        # Token buffer - keeps ALL tokens (not chunked)
        token_buffer = []
        total_tokens_generated = 0
        total_audio_chunks = 0

        # Generate tokens with vLLM (streaming)
        print(f"🔮 Starting token generation...")

        # Generate unique request ID for concurrent streaming support
        import uuid
        import time
        request_id = f"slide-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Stream tokens with sliding window
        async for request_output in results_generator:
            # Extract generated token IDs
            generated_ids = request_output.outputs[0].token_ids

            # Process only new tokens
            new_tokens = generated_ids[total_tokens_generated:]
            total_tokens_generated = len(generated_ids)

            # Filter and buffer SNAC tokens
            for token_id in new_tokens:
                if SNAC_MIN_ID <= token_id <= SNAC_MAX_ID:
                    token_buffer.append(token_id)

                    # Process every 7 tokens (1 frame) when we have enough for sliding window
                    # Official approach: once count > 27, take last 28 tokens
                    if len(token_buffer) % 7 == 0 and len(token_buffer) > 27:
                        # Sliding window: take last 28 tokens (4 frames)
                        window_tokens = token_buffer[-28:]

                        # Decode with sliding window mode (returns middle 2048 samples only)
                        if self.snac_decoder.enable_batching:
                            audio_bytes = await self.snac_decoder.decode_single_async(
                                window_tokens,
                                trim_warmup=False,  # Sliding window handles trimming
                                use_sliding_window=True  # CRITICAL: Use sliding window mode
                            )
                        else:
                            audio_bytes = self.snac_decoder.decode_to_bytes(
                                window_tokens,
                                trim_warmup=False,
                                use_sliding_window=True
                            )

                        if audio_bytes:
                            total_audio_chunks += 1
                            if total_audio_chunks == 1:
                                print(f"🎵 First chunk decoded ({len(audio_bytes)} bytes, sliding window)")
                            yield audio_bytes

        # Note: No final chunk processing needed - sliding window handles all tokens
        # as they come in (every 7 tokens after the first 28)

        print(f"✅ Sliding window streaming complete: {total_tokens_generated} tokens → {total_audio_chunks} audio chunks")

    def _extract_bicodec_tokens_from_text(self, text: str) -> tuple[List[int], List[int]]:
        """
        Extract BiCodec semantic and global tokens from generated text.

        Args:
            text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
        """
        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", text)
        semantic_ids = [int(t) for t in semantic_matches]

        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", text)
        global_ids = [int(t) for t in global_matches]

        return semantic_ids, global_ids

    async def generate_speech_stream_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,  # Added for Spark TTS parity with non-streaming
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech audio with sliding window streaming for Indic model.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "<laugh> Hello there!"
                - "नमस्ते! <excited> आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops
            seed: Random seed for reproducibility

        Yields:
            Audio bytes (int16 PCM, 16kHz mono - BiCodec)
        """
        import time
        t_start = time.time()

        # Build prompt using Indic prompt builder
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling (matching non-streaming pipeline parameters)
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID (128258)
        # which doesn't exist in the Spark vocab and would cause generation to run to max_tokens
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,  # Keep BiCodec tokens in output
            seed=seed,
        )

        # BiCodec token buffers (separate for semantic and global)
        semantic_buffer = []
        global_buffer = []
        processed_token_count = 0  # Track how many vLLM tokens we've parsed
        total_audio_chunks = 0

        # BiCodec streaming configuration
        EXPECTED_GLOBAL_COUNT = 32  # BiCodec uses 32 fixed global tokens (per paper)
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8, -120ms TTFB)
        # OPTIMIZATION: Increased from 24 to 48 to reduce O(n) re-decode calls by 2x.
        # At 50 TPS, 48 tokens = ~960ms audio per chunk. Cuts total decode calls roughly in half.
        DECODE_INTERVAL = 48
        CROSSFADE_MS = 50
        # OPTIMIZATION: Windowed decode - only decode last WINDOW_SIZE tokens instead of ALL.
        # BiCodec WaveGenerator has finite receptive field (~64 tokens based on kernel sizes).
        # Window must be >= receptive_field + DECODE_INTERVAL to avoid boundary artifacts.
        WINDOW_SIZE = 128  # 128 tokens covers receptive field + margin for crossfade

        # Generate tokens with vLLM (streaming)
        import uuid
        request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser

        t_gen_start = time.time()
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track timing
        t_first_token = None
        t_first_chunk_ready = None
        t_first_chunk_decoded = None
        iteration_count = 0
        last_decode_count = 0

        # Track audio samples EMITTED to the user (not just decoded)
        # BiCodec produces ~320 samples per semantic token at 16kHz
        # CRITICAL: This tracks samples that have been YIELDED, not held in tail
        total_samples_emitted_to_user = 0
        # Keep a tail to enable crossfading between consecutive emitted chunks
        previous_chunk_tail: Optional[bytes] = None

        # Stream tokens with BiCodec extraction
        async for request_output in results_generator:
            iteration_count += 1
            t_iter = time.time()

            # Track first token received
            if t_first_token is None:
                t_first_token = t_iter

            # Extract generated token IDs
            generated_ids = request_output.outputs[0].token_ids

            # OPTIMIZATION: Only process NEW tokens (incremental parsing)
            # Old code: tokenizer.decode(ALL) + re.findall(ALL) = O(n²) per stream
            # New code: parse only new tokens = O(1) per token
            new_token_ids = generated_ids[processed_token_count:]
            processed_token_count = len(generated_ids)

            # Parse new tokens and append to buffers (in-place modification)
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)

            # TRUE STREAMING with BiCodec two-phase generation:
            # Phase 1: Generate 32 global tokens (pre-roll)
            # Phase 2: Stream semantic tokens incrementally

            # Check if we have the 32 global tokens (pre-roll complete)
            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
                # Pre-roll complete! Now we can stream semantic tokens
                if t_first_chunk_ready is None and len(semantic_buffer) >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    t_first_chunk_ready = time.time()

                # Stream semantic tokens incrementally
                # Only decode if we have enough NEW semantic tokens since last decode
                semantic_count = len(semantic_buffer)
                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
                    # Check if we should decode (every DECODE_INTERVAL new semantic tokens)
                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
                        last_decode_count = semantic_count

                        # OPTIMIZATION: Windowed decode instead of full re-decode
                        # Only decode the last WINDOW_SIZE tokens (O(1) per chunk instead of O(n))
                        # For first few chunks when total tokens < WINDOW_SIZE, decode all
                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]

                        if len(semantic_buffer) <= WINDOW_SIZE:
                            # Early phase: decode all tokens (window not yet relevant)
                            decode_semantic = semantic_buffer
                        else:


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Analyzing streaming output kind for optimization**


I found the streaming hot path and I’m now drilling into the exact token loop internals. Next I’ll patch it for vLLM streaming-friendly output mode (`DELTA`) and remove cumulative-token overhead in the streaming generators.


cd /home/ubuntu/veenaModal && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '200,520p'


cd /home/ubuntu/veenaModal && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '520,880p'


Output:
Chunk ID: 7048f7
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 4452
Output:
   200	        Args:
   201	            text: Generated text containing BiCodec token markers
   202
   203	        Returns:
   204	            Tuple of (semantic_ids, global_ids)
   205	        """
   206	        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", text)
   207	        semantic_ids = [int(t) for t in semantic_matches]
   208
   209	        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", text)
   210	        global_ids = [int(t) for t in global_matches]
   211
   212	        return semantic_ids, global_ids
   213
   214	    async def generate_speech_stream_indic(
   215	        self,
   216	        speaker: str,
   217	        text: str,
   218	        temperature: float = DEFAULT_TEMPERATURE,
   219	        top_k: int = DEFAULT_TOP_K,  # Added for Spark TTS parity with non-streaming
   220	        top_p: float = DEFAULT_TOP_P,
   221	        max_tokens: int = DEFAULT_MAX_TOKENS,
   222	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   223	        seed: Optional[int] = None,
   224	    ) -> AsyncGenerator[bytes, None]:
   225	        """
   226	        Generate speech audio with sliding window streaming for Indic model.
   227
   228	        Yields audio chunks using overlapping windows for smooth playback.
   229
   230	        Args:
   231	            speaker: Speaker name (one of 12 predefined speakers)
   232	            text: Text to synthesize with inline emotion tags
   233	                Examples:
   234	                - "Hello! Welcome."
   235	                - "<laugh> Hello there!"
   236	                - "नमस्ते! <excited> आज का दिन बहुत अच्छा है।"
   237	            temperature: Sampling temperature
   238	            top_p: Nucleus sampling
   239	            max_tokens: Max SNAC tokens to generate
   240	            repetition_penalty: Prevent loops
   241	            seed: Random seed for reproducibility
   242
   243	        Yields:
   244	            Audio bytes (int16 PCM, 16kHz mono - BiCodec)
   245	        """
   246	        import time
   247	        t_start = time.time()
   248
   249	        # Build prompt using Indic prompt builder
   250	        prompt = self.prompt_builder.build_prefix(speaker, text)
   251
   252	        # Configure sampling (matching non-streaming pipeline parameters)
   253	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID (128258)
   254	        # which doesn't exist in the Spark vocab and would cause generation to run to max_tokens
   255	        sampling_params = SamplingParams(
   256	            temperature=temperature,
   257	            top_k=top_k,
   258	            top_p=top_p,
   259	            max_tokens=max_tokens,
   260	            min_tokens=DEFAULT_MIN_TOKENS,
   261	            repetition_penalty=repetition_penalty,
   262	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
   263	            skip_special_tokens=False,  # Keep BiCodec tokens in output
   264	            seed=seed,
   265	        )
   266
   267	        # BiCodec token buffers (separate for semantic and global)
   268	        semantic_buffer = []
   269	        global_buffer = []
   270	        processed_token_count = 0  # Track how many vLLM tokens we've parsed
   271	        total_audio_chunks = 0
   272
   273	        # BiCodec streaming configuration
   274	        EXPECTED_GLOBAL_COUNT = 32  # BiCodec uses 32 fixed global tokens (per paper)
   275	        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8, -120ms TTFB)
   276	        # OPTIMIZATION: Increased from 24 to 48 to reduce O(n) re-decode calls by 2x.
   277	        # At 50 TPS, 48 tokens = ~960ms audio per chunk. Cuts total decode calls roughly in half.
   278	        DECODE_INTERVAL = 48
   279	        CROSSFADE_MS = 50
   280	        # OPTIMIZATION: Windowed decode - only decode last WINDOW_SIZE tokens instead of ALL.
   281	        # BiCodec WaveGenerator has finite receptive field (~64 tokens based on kernel sizes).
   282	        # Window must be >= receptive_field + DECODE_INTERVAL to avoid boundary artifacts.
   283	        WINDOW_SIZE = 128  # 128 tokens covers receptive field + margin for crossfade
   284
   285	        # Generate tokens with vLLM (streaming)
   286	        import uuid
   287	        request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
   288
   289	        # Reuse pre-warmed token parser (singleton, created once in __init__)
   290	        token_parser = self.token_parser
   291
   292	        t_gen_start = time.time()
   293	        results_generator = self.model.engine.generate(
   294	            prompt=prompt,
   295	            sampling_params=sampling_params,
   296	            request_id=request_id,
   297	        )
   298
   299	        # Track timing
   300	        t_first_token = None
   301	        t_first_chunk_ready = None
   302	        t_first_chunk_decoded = None
   303	        iteration_count = 0
   304	        last_decode_count = 0
   305
   306	        # Track audio samples EMITTED to the user (not just decoded)
   307	        # BiCodec produces ~320 samples per semantic token at 16kHz
   308	        # CRITICAL: This tracks samples that have been YIELDED, not held in tail
   309	        total_samples_emitted_to_user = 0
   310	        # Keep a tail to enable crossfading between consecutive emitted chunks
   311	        previous_chunk_tail: Optional[bytes] = None
   312
   313	        # Stream tokens with BiCodec extraction
   314	        async for request_output in results_generator:
   315	            iteration_count += 1
   316	            t_iter = time.time()
   317
   318	            # Track first token received
   319	            if t_first_token is None:
   320	                t_first_token = t_iter
   321
   322	            # Extract generated token IDs
   323	            generated_ids = request_output.outputs[0].token_ids
   324
   325	            # OPTIMIZATION: Only process NEW tokens (incremental parsing)
   326	            # Old code: tokenizer.decode(ALL) + re.findall(ALL) = O(n²) per stream
   327	            # New code: parse only new tokens = O(1) per token
   328	            new_token_ids = generated_ids[processed_token_count:]
   329	            processed_token_count = len(generated_ids)
   330
   331	            # Parse new tokens and append to buffers (in-place modification)
   332	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
   333
   334	            # TRUE STREAMING with BiCodec two-phase generation:
   335	            # Phase 1: Generate 32 global tokens (pre-roll)
   336	            # Phase 2: Stream semantic tokens incrementally
   337
   338	            # Check if we have the 32 global tokens (pre-roll complete)
   339	            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   340	                # Pre-roll complete! Now we can stream semantic tokens
   341	                if t_first_chunk_ready is None and len(semantic_buffer) >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   342	                    t_first_chunk_ready = time.time()
   343
   344	                # Stream semantic tokens incrementally
   345	                # Only decode if we have enough NEW semantic tokens since last decode
   346	                semantic_count = len(semantic_buffer)
   347	                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   348	                    # Check if we should decode (every DECODE_INTERVAL new semantic tokens)
   349	                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
   350	                        last_decode_count = semantic_count
   351
   352	                        # OPTIMIZATION: Windowed decode instead of full re-decode
   353	                        # Only decode the last WINDOW_SIZE tokens (O(1) per chunk instead of O(n))
   354	                        # For first few chunks when total tokens < WINDOW_SIZE, decode all
   355	                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]
   356
   357	                        if len(semantic_buffer) <= WINDOW_SIZE:
   358	                            # Early phase: decode all tokens (window not yet relevant)
   359	                            decode_semantic = semantic_buffer
   360	                        else:
   361	                            # Windowed: only decode last WINDOW_SIZE tokens
   362	                            decode_semantic = semantic_buffer[-WINDOW_SIZE:]
   363
   364	                        t_decode_start = time.time()
   365	                        audio_bytes = await self.snac_decoder.decode_single_async(
   366	                            semantic_ids=decode_semantic,
   367	                            global_ids=window_global,
   368	                            trim_warmup=False,
   369	                            use_sliding_window=False,
   370	                        )
   371	                        t_decode_end = time.time()
   372
   373	                        if audio_bytes:
   374	                            total_samples_decoded = len(audio_bytes) // 2
   375
   376	                            if len(semantic_buffer) <= WINDOW_SIZE:
   377	                                # Early phase: extract new samples from full decode
   378	                                if total_samples_decoded > total_samples_emitted_to_user:
   379	                                    new_bytes_start = total_samples_emitted_to_user * 2
   380	                                    new_audio_bytes = audio_bytes[new_bytes_start:]
   381	                                else:
   382	                                    new_audio_bytes = b""
   383	                            else:
   384	                                # Windowed phase: take only the tail portion of the window decode
   385	                                # The last DECODE_INTERVAL tokens' worth of audio is the new content
   386	                                # Each token produces ~320 samples at 16kHz/50TPS
   387	                                new_samples_approx = DECODE_INTERVAL * 320
   388	                                if total_samples_decoded > new_samples_approx:
   389	                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
   390	                                else:
   391	                                    new_bytes_start = 0
   392	                                new_audio_bytes = audio_bytes[new_bytes_start:]
   393
   394	                            if new_audio_bytes:
   395	                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   396	                                    previous_chunk_tail,
   397	                                    new_audio_bytes,
   398	                                    sample_rate_hz=16000,
   399	                                    crossfade_ms=CROSSFADE_MS,
   400	                                )
   401
   402	                                samples_emitted_in_this_chunk = len(to_emit) // 2
   403	                                total_samples_emitted_to_user += samples_emitted_in_this_chunk
   404
   405	                                if to_emit:
   406	                                    total_audio_chunks += 1
   407	                                    if t_first_chunk_decoded is None:
   408	                                        t_first_chunk_decoded = t_decode_end
   409
   410	                                    yield to_emit
   411
   412	        # Final chunk: decode any remaining semantic tokens
   413	        # Use the same 32 global tokens (they get pooled)
   414	        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   415	            audio_bytes = self.snac_decoder.decode_streaming(
   416	                semantic_ids=semantic_buffer,  # All semantic tokens
   417	                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],  # Use first 32 global tokens
   418	                use_sliding_window=False,  # Get all samples
   419	                trim_warmup=False
   420	            )
   421	            if audio_bytes:
   422	                # Only yield NEW samples
   423	                total_samples_decoded = len(audio_bytes) // 2
   424
   425	                if total_samples_decoded > total_samples_emitted_to_user:
   426	                    new_bytes_start = total_samples_emitted_to_user * 2
   427	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   428
   429	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   430	                        previous_chunk_tail,
   431	                        new_audio_bytes,
   432	                        sample_rate_hz=16000,
   433	                        crossfade_ms=CROSSFADE_MS,
   434	                    )
   435
   436	                    samples_emitted_in_final = len(to_emit) // 2
   437	                    total_samples_emitted_to_user += samples_emitted_in_final
   438
   439	                    if to_emit:
   440	                        total_audio_chunks += 1
   441	                        yield to_emit
   442
   443	        # Flush any remaining tail that was held back for crossfade
   444	        if previous_chunk_tail:
   445	            total_audio_chunks += 1
   446	            tail_samples = len(previous_chunk_tail) // 2
   447	            total_samples_emitted_to_user += tail_samples
   448	            yield previous_chunk_tail
   449
   450	        # Log completion summary
   451	        t_end = time.time()
   452	        audio_duration_s = total_samples_emitted_to_user / 16000  # 16kHz sample rate
   453	        ttfb_ms = (t_first_chunk_decoded - t_start) * 1000 if t_first_chunk_decoded else 0
   454	        print(f"🎵 Streaming complete: {audio_duration_s:.2f}s audio, TTFB: {ttfb_ms:.0f}ms, {total_audio_chunks} chunks")
   455
   456	    async def generate_speech_stream_indic_first_chunk(
   457	        self,
   458	        speaker: str,
   459	        text: str,
   460	        temperature: float = DEFAULT_TEMPERATURE,
   461	        top_k: int = DEFAULT_TOP_K,
   462	        top_p: float = DEFAULT_TOP_P,
   463	        max_tokens: int = DEFAULT_MAX_TOKENS,
   464	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   465	        seed: Optional[int] = None,
   466	    ) -> AsyncGenerator[tuple, None]:
   467	        """
   468	        Generate speech for the FIRST chunk of multi-chunk text, capturing global tokens.
   469
   470	        This method is specifically for chunked generation: it yields (audio_bytes, global_ids)
   471	        tuples instead of just audio_bytes. The caller captures global_ids from the first
   472	        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.
   473
   474	        Use Case (chunked streaming with voice consistency):
   475	            globals_captured = None
   476	            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
   477	                if globals_captured is None and global_ids:
   478	                    globals_captured = global_ids  # Capture once
   479	                yield audio_bytes
   480	            # Now use globals_captured for subsequent chunks
   481
   482	        Args:
   483	            speaker: Speaker name
   484	            text: First text chunk to synthesize
   485	            temperature: Sampling temperature
   486	            top_k: Top-k sampling
   487	            top_p: Nucleus sampling
   488	            max_tokens: Max tokens to generate
   489	            repetition_penalty: Prevent repetition
   490	            seed: Random seed for reproducibility
   491
   492	        Yields:
   493	            Tuple of (audio_bytes, global_ids)
   494	            - audio_bytes: Raw PCM audio (int16, 16kHz)
   495	            - global_ids: List of 32 global token IDs (populated after first decode, else empty)
   496
   497	        Thread Safety:
   498	            This method is stateless and thread-safe. Each call creates its own
   499	            request-scoped state. No global state is modified or shared.
   500	        """
   501	        import time
   502	        t_start = time.time()
   503
   504	        # Build prompt using Indic prompt builder
   505	        prompt = self.prompt_builder.build_prefix(speaker, text)
   506
   507	        # Configure sampling
   508	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
   509	        sampling_params = SamplingParams(
   510	            temperature=temperature,
   511	            top_k=top_k,
   512	            top_p=top_p,
   513	            max_tokens=max_tokens,
   514	            min_tokens=DEFAULT_MIN_TOKENS,
   515	            repetition_penalty=repetition_penalty,
   516	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
   517	            skip_special_tokens=False,
   518	            seed=seed,
   519	        )
   520