Output:
Chunk ID: 1c0d8e
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 39100
Output:
Total output lines: 1628

scripts/detailed_word_analysis.py:10:5. Allow user to verify if it's ASR error or TTS chunking error
scripts/detailed_word_analysis.py:15:- Or if chunk was skipped (TTS error)
scripts/detailed_word_analysis.py:158:    print("🎵 Generating audio (this will take 50-70s with chunking)...")
scripts/detailed_word_analysis.py:181:    chunked = response.headers.get('X-Text-Chunked', 'unknown')
scripts/detailed_word_analysis.py:186:    print(f"   Chunked: {chunked}")
scripts/detailed_word_analysis.py:251:                print(f"   Gap #{idx}: {gap_sec:.1f}s between chunks")
scripts/detailed_word_analysis.py:294:            print(f"      If you DON'T hear it → TTS error (chunk was skipped/incomplete)")
scripts/detailed_word_analysis.py:302:    # Also check for repeated words (might indicate chunking overlap issues)
scripts/detailed_word_analysis.py:318:        print("🔍 REPEATED WORDS (may indicate chunk overlap):")
scripts/detailed_word_analysis.py:334:        f.write(f"Chunked: {chunked}\n")
scripts/detailed_word_analysis.py:407:        print("   4. If you DON'T hear it → TTS error (chunk skipped)")
scripts/detailed_word_analysis.py:414:        print("   - All chunks were processed ✅")
scripts/detailed_word_analysis.py:431:        print(f"   Missing: {len(missing_words)} words (likely ASR errors, not chunking errors)")
scripts/detailed_word_analysis.py:437:        print(f"   Missing: {len(missing_words)} words - some may be skipped chunks")
scripts/detailed_word_analysis.py:440:        print(f"   Missing: {len(missing_words)} words - likely chunking issues")
scripts/measure_ttfb_detailed_v2.py:3:Measure TTFB with detailed per-chunk timing
scripts/measure_ttfb_detailed_v2.py:11:def measure_streaming_with_all_chunks(text="Hello world, this is a test.", speaker="lipakshi"):
scripts/measure_ttfb_detailed_v2.py:13:    Measure TTFB showing ALL chunks received
scripts/measure_ttfb_detailed_v2.py:40:    chunk_times = []
scripts/measure_ttfb_detailed_v2.py:41:    chunk_sizes = []
scripts/measure_ttfb_detailed_v2.py:44:    print(f"\n📦 Receiving chunks:")
scripts/measure_ttfb_detailed_v2.py:45:    for i, chunk in enumerate(response.iter_content(chunk_size=None)):
scripts/measure_ttfb_detailed_v2.py:46:        if chunk:
scripts/measure_ttfb_detailed_v2.py:47:            t_chunk = time.time() - t0
scripts/measure_ttfb_detailed_v2.py:48:            chunk_times.append(t_chunk)
scripts/measure_ttfb_detailed_v2.py:49:            chunk_sizes.append(len(chunk))
scripts/measure_ttfb_detailed_v2.py:50:            total_bytes += len(chunk)
scripts/measure_ttfb_detailed_v2.py:52:            # Show first 10 chunks and then every 10th
scripts/measure_ttfb_detailed_v2.py:55:                if i == 0 and chunk.startswith(b'RIFF'):
scripts/measure_ttfb_detailed_v2.py:60:                print(f"   #{i+1:3d} @ T+{t_chunk:.3f}s: {len(chunk):>6,} bytes{content_type}")
scripts/measure_ttfb_detailed_v2.py:65:    print(f"   ├─ Total chunks:          {len(chunk_times)}")
scripts/measure_ttfb_detailed_v2.py:67:    print(f"   ├─ First chunk (TTFB):    T+{chunk_times[0]:.3f}s = {chunk_times[0]*1000:.0f}ms")
scripts/measure_ttfb_detailed_v2.py:68:    if len(chunk_times) > 1:
scripts/measure_ttfb_detailed_v2.py:69:        print(f"   ├─ Second chunk:          T+{chunk_times[1]:.3f}s = {chunk_times[1]*1000:.0f}ms")
scripts/measure_ttfb_detailed_v2.py:70:        print(f"   ├─ Delta (chunk 1→2):     {(chunk_times[1] - chunk_times[0])*1000:.0f}ms")
scripts/measure_ttfb_detailed_v2.py:74:    if chunk_sizes[0] == 44:
scripts/measure_ttfb_detailed_v2.py:75:        print(f"   First chunk is WAV header (44 bytes) - true audio starts at chunk #2")
scripts/measure_ttfb_detailed_v2.py:76:        if len(chunk_times) > 1:
scripts/measure_ttfb_detailed_v2.py:77:            print(f"   TRUE HEARABLE AUDIO TTFB: {chunk_times[1]*1000:.0f}ms")
scripts/measure_ttfb_detailed_v2.py:79:        print(f"   First chunk contains audio data")
scripts/measure_ttfb_detailed_v2.py:80:        print(f"   TRUE HEARABLE AUDIO TTFB: {chunk_times[0]*1000:.0f}ms")
scripts/measure_ttfb_detailed_v2.py:84:    measure_streaming_with_all_chunks()
scripts/elevenlabs_asr_analysis.py:8:- TTS chunking error (words not spoken at all)
scripts/elevenlabs_asr_analysis.py:198:        print(f"\n   ✅ PERFECT - No chunking errors!")
scripts/elevenlabs_asr_analysis.py:204:        print(f"\n   ❌ POOR - >10% missing (likely chunking errors)")
scripts/create_visual_map.py:4:- Text broken into chunks
scripts/create_visual_map.py:5:- Timestamps for each chunk
scripts/create_visual_map.py:64:# Simple chunking (split into ~500 char chunks at sentence boundaries)
scripts/create_visual_map.py:65:def simple_chunk(text, max_length=500):
scripts/create_visual_map.py:66:    """Simple sentence-aware chunking."""
scripts/create_visual_map.py:68:    chunks = []
scripts/create_visual_map.py:69:    current_chunk = ""
scripts/create_visual_map.py:76:        if len(current_chunk) + len(full_sent) > max_length and current_chunk:
scripts/create_visual_map.py:77:            chunks.append(current_chunk.strip())
scripts/create_visual_map.py:78:            current_chunk = full_sent
scripts/create_visual_map.py:80:            current_chunk += full_sent
scripts/create_visual_map.py:82:    if current_chunk.strip():
scripts/create_visual_map.py:83:        chunks.append(current_chunk.strip())
scripts/create_visual_map.py:85:    return chunks
scripts/create_visual_map.py:87:chunks = simple_chunk(REFERENCE_TEXT, max_length=500)
scripts/create_visual_map.py:89:print(f"📝 Text chunked into {len(chunks)} chunks")
scripts/create_visual_map.py:113:# Analyze each chunk
scripts/create_visual_map.py:114:print("🔍 Analyzing chunks...")
scripts/create_visual_map.py:119:def get_words_in_chunk(chunk):
scripts/create_visual_map.py:120:    clean = clean_text(chunk)
scripts/create_visual_map.py:126:# Estimate timestamps for chunks (based on audio duration)
scripts/create_visual_map.py:128:total_chars = sum(len(c) for c in chunks)
scripts/create_visual_map.py:131:chunk_data = []
scripts/create_visual_map.py:132:for i, chunk in enumerate(chunks):
scripts/create_visual_map.py:133:    chunk_words = get_words_in_chunk(chunk)
scripts/create_visual_map.py:135:    # Estimate chunk duration
scripts/create_visual_map.py:136:    chunk_duration = (len(chunk) / total_chars) * audio_duration
scripts/create_visual_map.py:138:    end_time = current_time + chunk_duration
scripts/create_visual_map.py:142:    missing_words = [w for w in chunk_words if w not in trans_word_set]
scripts/create_visual_map.py:143:    found_words = [w for w in chunk_words if w in trans_word_set]
scripts/create_visual_map.py:145:    chunk_data.append({
scripts/create_visual_map.py:147:        'text': chunk,
scripts/create_visual_map.py:148:        'clean': clean_text(chunk),
scripts/create_visual_map.py:149:        'words': chunk_words,
scripts/create_visual_map.py:154:        'duration': chunk_duration,
scripts/create_visual_map.py:155:        'accuracy': len(found_words) / len(chunk_words) * 100 if chunk_words else 0
scripts/create_visual_map.py:158:print(f"✅ Analyzed {len(chunk_data)} chunks")
scripts/create_visual_map.py:211:        .chunk {
scripts/create_visual_map.py:217:        .chunk-header {
scripts/create_visual_map.py:225:        .chunk-title {
scripts/create_visual_map.py:229:        .chunk-stats {
scripts/create_visual_map.py:234:        .chunk-body {
scripts/create_visual_map.py:238:        .chunk-text {
scripts/create_visual_map.py:302:        .timeline {
scripts/create_visual_map.py:343:
""" + str(len(chunks)) + """

scripts/create_visual_map.py:351:
""" + str(sum(len(c['words']) for c in chunk_data)) + """

scripts/create_visual_map.py:355:
""" + str(sum(len(c['missing']) for c in chunk_data)) + """

scripts/create_visual_map.py:375:# Add each chunk
scripts/create_visual_map.py:376:for chunk_info in chunk_data:
scripts/create_visual_map.py:378:    clean = chunk_info['clean']
scripts/create_visual_map.py:379:    words = chunk_info['words']
scripts/create_visual_map.py:380:    missing = set(chunk_info['missing'])
scripts/create_visual_map.py:402:    text_with_tags = chunk_info['text']
scripts/create_visual_map.py:410:    accuracy = chunk_info['accuracy']
scripts/create_visual_map.py:411:    missing_count = len(chunk_info['missing'])
scripts/create_visual_map.py:414:

scripts/create_visual_map.py:415:

scripts/create_visual_map.py:416:
Chunk {chunk_info['index']} / {len(chunks)}

scripts/create_visual_map.py:417:

scripts/create_visual_map.py:418:                    ⏱ {chunk_info['start_time']:.1f}s - {chunk_info['end_time']:.1f}s
scripts/create_visual_map.py:419:                    📝 {len(chunk_info['words'])} words
scripts/create_visual_map.py:423:

scripts/create_visual_map.py:437:
{highlighted}

scripts/create_visual_map.py:445:                    {''.join(f'{w}' for w in chunk_info['missing'])}
scripts/create_visual_map.py:455:best_chunks = sorted(chunk_data, key=lambda x: x['accuracy'], reverse=True)[:3]
scripts/create_visual_map.py:456:worst_chunks = sorted(chunk_data, key=lambda x: x['accuracy'])[:3]
scripts/create_visual_map.py:466:for chunk in best_chunks:
scripts/create_visual_map.py:468:
Chunk {chunk['index']}: {chunk['accuracy']:.1f}% accuracy
scripts/create_visual_map.py:469:                    ({chunk['start_time']:.1f}s - {chunk['end_time']:.1f}s)

scripts/create_visual_map.py:479:for chunk in worst_chunks:
scripts/create_visual_map.py:481:
Chunk {chunk['index']}: {chunk['accuracy']:.1f}% accuracy,
scripts/create_visual_map.py:482:                    {len(chunk['missing'])} missing words
scripts/create_visual_map.py:483:                    ({chunk['start_time']:.1f}s - {chunk['end_time']:.1f}s)

scripts/create_visual_map.py:495:output_file = "visual_chunking_map.html"
scripts/create_visual_map.py:502:print(f"   Total chunks: {len(chunks)}")
scripts/create_visual_map.py:503:print(f"   Total words: {sum(len(c['words']) for c in chunk_data)}")
scripts/create_visual_map.py:504:print(f"   Missing words: {sum(len(c['missing']) for c in chunk_data)}")
scripts/create_visual_map.py:505:print(f"   Average accuracy: {sum(c['accuracy'] for c in chunk_data)/len(chunk_data):.1f}%")
veena3modal/core/token_utils.py:4:Provides O(1) per-token parsing instead of O(n) decode + regex scanning.
veena3modal/core/token_utils.py:7:    tokenizer.decode(all_tokens)  # O(n)
veena3modal/core/token_utils.py:11:    for new_token_id in new_tokens:  # Only new tokens
veena3modal/core/token_utils.py:23:    Caching parser for BiCodec tokens.
veena3modal/core/token_utils.py:27:    BiCodec tokens in the vocabulary look like:
veena3modal/core/token_utils.py:54:            tokenizer: HuggingFace tokenizer with BiCodec special tokens
veena3modal/core/token_utils.py:67:        Pre-populate cache with BiCodec tokens from vocabulary.
veena3modal/core/token_utils.py:109:        O(1) for cached tokens, O(1) amortized for uncached.
veena3modal/core/token_utils.py:121:        # Cache miss - decode and parse
veena3modal/core/token_utils.py:123:            token_str = self.tokenizer.decode([token_id], skip_special_tokens=False)
veena3modal/core/token_utils.py:139:        Parse new tokens and append to existing buffers.
veena3modal/core/token_utils.py:142:        - Only processes NEW tokens (not entire history)
veena3modal/core/token_utils.py:176:            "bicodec_tokens": bicodec_count,
veena3modal/core/token_utils.py:177:            "non_bicodec_tokens": total - bicodec_count,
scripts/stress_test_production.py:160:    ttfb_ms: float = 0.0
scripts/stress_test_production.py:182:                ttfb = 0.0
scripts/stress_test_production.py:184:                async for chunk in resp.aiter_bytes():
scripts/stress_test_production.py:186:                        ttfb = (time.time() - start) * 1000
scripts/stress_test_production.py:188:                    total += len(chunk)
scripts/stress_test_production.py:192:                    return RequestResult(True, 200, latency, ttfb, total,
scripts/stress_test_production.py:194:                return RequestResult(False, resp.status_code, latency, ttfb,
scripts/stress_test_production.py:258:    ttfbs = [r.ttfb_ms for r in successes if r.ttfb_ms > 0]
scripts/stress_test_production.py:285:    if ttfbs and stream:
scripts/stress_test_production.py:287:        print(f"    min={min(ttfbs):.0f}  avg={statistics.mean(ttfbs):.0f}  "
scripts/stress_test_production.py:288:              f"p50={percentile(ttfbs,50):.0f}  p95={percentile(ttfbs,95):.0f}  max={max(ttfbs):.0f}")
scripts/stress_test_production.py:313:        "ttfb_ms": {"avg": statistics.mean(ttfbs), "p50": percentile(ttfbs, 50),
scripts/stress_test_production.py:314:                     "p95": percentile(ttfbs, 95), "max": max(ttfbs)} if ttfbs else {},
scripts/stress_test_production.py:388:            t = r.get("ttfb_ms", {})
scripts/validate_modal_endpoint.py:63:def test_metrics():
scripts/validate_modal_endpoint.py:64:    """Test metrics endpoint."""
scripts/validate_modal_endpoint.py:67:    url = f"{ENDPOINT_URL}/v1/tts/metrics"
scripts/validate_modal_endpoint.py:75:        print("Sample metrics:")
scripts/validate_modal_endpoint.py:142:        print(f"TTFB: {headers.get('x-ttfb-ms', 'N/A')}ms")
scripts/validate_modal_endpoint.py:163:    text = "This is a test of the streaming text to speech functionality. It should send audio chunks progressively."
scripts/validate_modal_endpoint.py:169:    first_chunk_time = None
scripts/validate_modal_endpoint.py:171:    chunk_count = 0
scripts/validate_modal_endpoint.py:190:        # Collect chunks
scripts/validate_modal_endpoint.py:192:        for chunk in response.iter_bytes(chunk_size=4096):
scripts/validate_modal_endpoint.py:193:            if chunk:
scripts/validate_modal_endpoint.py:194:                chunk_count += 1
scripts/validate_modal_endpoint.py:195:                total_bytes += len(chunk)
scripts/validate_modal_endpoint.py:196:                audio_data += chunk
scripts/validate_modal_endpoint.py:198:                if first_chunk_time is None:
scripts/validate_modal_endpoint.py:199:                    first_chunk_time = time.time()
scripts/validate_modal_endpoint.py:200:                    ttfb = (first_chunk_time - start) * 1000
scripts/validate_modal_endpoint.py:201:                    print(f"First chunk received! TTFB: {ttfb:.0f}ms")
scripts/validate_modal_endpoint.py:205:    print(f"Total chunks: {chunk_count}")
scripts/validate_modal_endpoint.py:247:            # Receive some audio chunks
scripts/validate_modal_endpoint.py:248:            chunk_count = 0
scripts/validate_modal_endpoint.py:255:                        chunk_count += 1
scripts/validate_modal_endpoint.py:260:                            print(f"Complete: {data.get('metrics', {})}")
scripts/validate_modal_endpoint.py:263:                            print(f"Progress: chunks={data.get('chunks_sent')}")
scripts/validate_modal_endpoint.py:268:            print(f"Total chunks: {chunk_count}")
scripts/validate_modal_endpoint.py:293:        results["metrics"] = test_metrics()
scripts/validate_modal_endpoint.py:296:        results["metrics"] = False
scripts/validate_true_streaming.py:8:3. Incremental chunk delivery
scripts/validate_true_streaming.py:21:def test_streaming(text, speaker, test_name, expected_ttfb_max=1000):
scripts/validate_true_streaming.py:44:    chunks = []
scripts/validate_true_streaming.py:46:    t_first_chunk = None
scripts/validate_true_streaming.py:48:    for chunk in response.iter_content(chunk_size=4096):
scripts/validate_true_streaming.py:49:        if chunk:
scripts/validate_true_streaming.py:50:            t_chunk = time.time()
scripts/validate_true_streaming.py:51:            if t_first_chunk is None:
scripts/validate_true_streaming.py:52:                t_first_chunk = t_chunk
scripts/validate_true_streaming.py:53:            chunks.append((t_chunk - t_start, len(chunk)))
scripts/validate_true_streaming.py:54:            total_bytes += len(chunk)
scripts/validate_true_streaming.py:57:    ttfb = (t_first_chunk - t_start) * 1000 if t_first_chunk else None
scripts/validate_true_streaming.py:61:    print(f"   TTFB: {ttfb:.0f}ms" if ttfb else "   TTFB: N/A")
scripts/validate_true_streaming.py:63:    print(f"   Chunks: {len(chunks)}")
scripts/validate_true_streaming.py:67:    if len(chunks) > 5:
scripts/validate_true_streaming.py:68:        chunk_times = [t for t, _ in chunks[:10]]
scripts/validate_true_streaming.py:69:        intervals = [chunk_times[i+1] - chunk_times[i] for i in range(len(chunk_times)-1)]
scripts/validate_true_streaming.py:81:    if ttfb is None:
scripts/validate_true_streaming.py:84:    elif ttfb > expected_ttfb_max:
scripts/validate_true_streaming.py:85:        print(f"\n⚠  WARNING: TTFB {ttfb:.0f}ms exceeds target {expected_ttfb_max}ms")
scripts/validate_true_streaming.py:87:        print(f"\n✅ PASSED: TTFB {ttfb:.0f}ms within target")
scripts/validate_true_streaming.py:92:    elif len(chunks) < 3:
scripts/validate_true_streaming.py:93:        print(f"⚠  WARNING: Only {len(chunks)} chunks (expected more for streaming)")
scripts/validate_true_streaming.py:95:        print(f"✅ PASSED: {len(chunks)} chunks streamed incrementally")
scripts/validate_true_streaming.py:97:    return success, ttfb, total_bytes, len(chunks)
scripts/validate_true_streaming.py:107:    success, ttfb, bytes_gen, chunks = test_streaming(
scripts/validate_true_streaming.py:112:    results.append(("Short English", success, ttfb, bytes_gen, chunks))
scripts/validate_true_streaming.py:115:    success, ttfb, bytes_gen, chunks = test_streaming(
scripts/validate_true_streaming.py:116:        text="This is a longer English text to test streaming with more content. We want to verify that audio chunks are generated and delivered incrementally, allowing the user to start hearing audio while the model is still generating the rest of the speech.",
scripts/validate_true_streaming.py:120:    results.append(("Longer English", success, ttfb, bytes_gen, chunks))
scripts/validate_true_streaming.py:123:    success, ttfb, bytes_gen, chunks = test_streaming(
scripts/validate_true_streaming.py:127:        expected_ttfb_max=1500  # Longer text, allow more time
scripts/validate_true_streaming.py:129:    results.append(("Hindi/Telugu", success, ttfb, bytes_gen, chunks))
scripts/validate_true_streaming.py:132:    success, ttfb, bytes_gen, chunks = test_streaming(
scripts/validate_true_streaming.py:137:    results.append(("Multi-Emotion", success, ttfb, bytes_gen, chunks))
scripts/validate_true_streaming.py:146:    for name, success, ttfb, bytes_gen, chunks in results:
scripts/validate_true_streaming.py:148:        ttfb_str = f"{ttfb:.0f}" if ttfb else "N/A"
scripts/validate_true_streaming.py:150:        print(f"{name:<20} {status:<10} {ttfb_str:<12} {kb:<12.1f} {chunks:<10}")
veena3modal/api/fastapi_app.py:30:# Prometheus metrics
veena3modal/api/fastapi_app.py:31:from veena3modal.shared.metrics import (
veena3modal/api/fastapi_app.py:32:    record_request_received as metrics_request_received,
veena3modal/api/fastapi_app.py:33:    record_request_completed as metrics_request_completed,
veena3modal/api/fastapi_app.py:34:    record_request_failed as metrics_request_failed,
veena3modal/api/fastapi_app.py:35:    record_ttfb,
veena3modal/api/fastapi_app.py:38:    record_chunks_sent,
veena3modal/api/fastapi_app.py:39:    get_metrics_text,
veena3modal/api/fastapi_app.py:40:    get_content_type as get_metrics_content_type,
veena3modal/api/fastapi_app.py:115:def _build_perf_headers(metrics: Dict[str, Any], request_inflight: int) -> Dict[str, str]:
veena3modal/api/fastapi_app.py:123:    if not isinstance(metrics, dict):
veena3modal/api/fastapi_app.py:124:        metrics = {}
veena3modal/api/fastapi_app.py:130:    # Expose selected metrics as explicit headers.
veena3modal/api/fastapi_app.py:140:        "x-timeline-first-batch-ms": "timeline_llm_first_batch_ms",
veena3modal/api/fastapi_app.py:141:        "x-timeline-llm-done-ms": "timeline_llm_done_ms",
veena3modal/api/fastapi_app.py:142:        "x-timeline-bicodec-done-ms": "timeline_bicodec_done_ms",
veena3modal/api/fastapi_app.py:143:        "x-timeline-total-ms": "timeline_total_ms",
veena3modal/api/fastapi_app.py:154:        "x-llm-decode-wall-ms": "llm_decode_wall_ms_total",
veena3modal/api/fastapi_app.py:155:        "x-llm-decode-gpu-ms": "llm_decode_gpu_ms_total",
veena3modal/api/fastapi_app.py:156:        "x-llm-decode-wall-ms-min": "llm_decode_wall_ms_min",
veena3modal/api/fastapi_app.py:157:        "x-llm-decode-wall-ms-max": "llm_decode_wall_ms_max",
veena3modal/api/fastapi_app.py:158:        "x-llm-decode-wall-ms-p50": "llm_decode_wall_ms_p50",
veena3modal/api/fastapi_app.py:159:        "x-llm-decode-gpu-ms-min": "llm_decode_gpu_ms_min",
veena3modal/api/fastapi_app.py:160:        "x-llm-decode-gpu-ms-max": "llm_decode_gpu_ms_max",
veena3modal/api/fastapi_app.py:161:        "x-llm-decode-gpu-ms-p50": "llm_decode_gpu_ms_p50",
veena3modal/api/fastapi_app.py:162:        "x-llm-decode-calls": "llm_decode_calls",
veena3modal/api/fastapi_app.py:163:        "x-llm-decode-cpu-ms": "bicodec_decode_cpu_ms",
veena3modal/api/fastapi_app.py:166:        "x-llm-tokens-per-batch-min": "tokens_per_batch_min",
veena3modal/api/fastapi_app.py:167:        "x-llm-tokens-per-batch-max": "tokens_per_batch_max",
veena3modal/api/fastapi_app.py:168:        "x-llm-tokens-per-batch-p50": "tokens_per_batch_p50",
veena3modal/api/fastapi_app.py:172:        "x-llm-time-in-queue-ms": "llm_time_in_queue_ms",
veena3modal/api/fastapi_app.py:173:        "x-llm-scheduler-ms": "llm_scheduler_ms",
veena3modal/api/fastapi_app.py:176:        "x-llm-text-chunked": "text_chunked",
veena3modal/api/fastapi_app.py:177:        "x-llm-chunks-processed": "chunks_processed",
veena3modal/api/fastapi_app.py:178:        "x-bicodec-decode-wall-ms": "bicodec_decode_wall_ms",
veena3modal/api/fastapi_app.py:179:        "x-bicodec-decode-gpu-ms": "bicodec_decode_gpu_ms",
veena3modal/api/fastapi_app.py:180:        "x-bicodec-decode-cpu-ms": "bicodec_decode_cpu_ms",
veena3modal/api/fastapi_app.py:184:        metrics = dict(metrics)
veena3modal/api/fastapi_app.py:185:        metrics["request_inflight"] = request_inflight
veena3modal/api/fastapi_app.py:188:        if metric_key not in metrics:
veena3modal/api/fastapi_app.py:190:        value = metrics.get(metric_key)
veena3modal/api/fastapi_app.py:207:        "timeline_markers",
veena3modal/api/fastapi_app.py:208:        "timeline_prompt_ready_ms",
veena3modal/api/fastapi_app.py:209:        "timeline_sampling_ready_ms",
veena3modal/api/fastapi_app.py:210:        "timeline_llm_first_batch_ms",
veena3modal/api/fastapi_app.py:211:        "timeline_llm_done_ms",
veena3modal/api/fastapi_app.py:212:        "timeline_parse_done_ms",
veena3modal/api/fastapi_app.py:213:        "timeline_validation_done_ms",
veena3modal/api/fastapi_app.py:214:        "timeline_bicodec_done_ms",
veena3modal/api/fastapi_app.py:215:        "timeline_wav_done_ms",
veena3modal/api/fastapi_app.py:216:        "timeline_request_done_ms",
veena3modal/api/fastapi_app.py:217:        "timeline_to_first_batch_ms",
veena3modal/api/fastapi_app.py:218:        "timeline_first_batch_to_llm_done_ms",
veena3modal/api/fastapi_app.py:219:        "timeline_post_llm_ms",
veena3modal/api/fastapi_app.py:220:        "timeline_parse_to_bicodec_done_ms",
veena3modal/api/fastapi_app.py:221:        "timeline_bicodec_to_request_done_ms",
veena3modal/api/fastap…27100 tokens truncated…_local.py:567:        chunking=chunking,
scripts/stress_test_optimized_local.py:586:                chunking=chunking,
scripts/stress_test_optimized_local.py:587:                max_tokens=max_tokens,
scripts/stress_test_optimized_local.py:629:    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}, chunking={level.chunking}")
scripts/stress_test_optimized_local.py:649:    ttfb = level.ttfb_stats
scripts/stress_test_optimized_local.py:650:    if ttfb and not level.mode.startswith("stream"):
scripts/stress_test_optimized_local.py:653:            f"    min={ttfb['min']:.0f} avg={ttfb['avg']:.0f} "
scripts/stress_test_optimized_local.py:654:            f"p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}"
scripts/stress_test_optimized_local.py:674:        decode = level.timing_stats("llm_decode_wall_ms_total")
scripts/stress_test_optimized_local.py:675:        if decode:
scripts/stress_test_optimized_local.py:677:            print(f"    avg={decode['avg']:.0f} p95={decode['p95']:.0f}")
scripts/stress_test_optimized_local.py:678:        bdecode = level.timing_stats("bicodec_decode_wall_ms")
scripts/stress_test_optimized_local.py:679:        if bdecode:
scripts/stress_test_optimized_local.py:680:            print("\n  BiCodec decode (ms):")
scripts/stress_test_optimized_local.py:681:            print(f"    avg={bdecode['avg']:.0f} p95={bdecode['p95']:.0f}")
scripts/stress_test_optimized_local.py:682:        bdecode_gpu = level.timing_stats("bicodec_decode_gpu_ms")
scripts/stress_test_optimized_local.py:683:        if bdecode_gpu:
scripts/stress_test_optimized_local.py:684:            print("\n  BiCodec decode GPU (ms):")
scripts/stress_test_optimized_local.py:685:            print(f"    avg={bdecode_gpu['avg']:.0f} p95={bdecode_gpu['p95']:.0f}")
scripts/stress_test_optimized_local.py:713:                    f"    client+queue overhead avg={client_overhead_avg:.0f} p95={client_overhead_p95:.0f}"
scripts/stress_test_optimized_local.py:716:        tl_first = level.timing_stats("timeline_llm_first_batch_ms")
scripts/stress_test_optimized_local.py:717:        tl_llm_done = level.timing_stats("timeline_llm_done_ms")
scripts/stress_test_optimized_local.py:718:        tl_parse_done = level.timing_stats("timeline_parse_done_ms")
scripts/stress_test_optimized_local.py:719:        tl_bicodec_done = level.timing_stats("timeline_bicodec_done_ms")
scripts/stress_test_optimized_local.py:720:        tl_total = level.timing_stats("timeline_total_ms")
scripts/stress_test_optimized_local.py:734:            bicodec_avg = bdecode.get("avg", 0.0) if bdecode else 0.0
scripts/stress_test_optimized_local.py:746:            timeline_samples = [
scripts/stress_test_optimized_local.py:748:                if isinstance(r.timing.get("timeline_total_ms"), (int, float))
scripts/stress_test_optimized_local.py:750:            if timeline_samples:
scripts/stress_test_optimized_local.py:751:                sample = sorted(timeline_samples, key=lambda r: r.latency_ms)[len(timeline_samples) // 2]
scripts/stress_test_optimized_local.py:755:                        float(sample.timing.get("timeline_llm_first_batch_ms", 0.0)),
scripts/stress_test_optimized_local.py:756:                        float(sample.timing.get("timeline_llm_done_ms", 0.0)),
scripts/stress_test_optimized_local.py:757:                        float(sample.timing.get("timeline_parse_done_ms", 0.0)),
scripts/stress_test_optimized_local.py:758:                        float(sample.timing.get("timeline_bicodec_done_ms", 0.0)),
scripts/stress_test_optimized_local.py:759:                        float(sample.timing.get("timeline_total_ms", 0.0)),
scripts/stress_test_optimized_local.py:821:            "chunking": level.chunking,
scripts/stress_test_optimized_local.py:827:            "ttfb_ms": level.ttfb_stats,
scripts/stress_test_optimized_local.py:838:                "llm_decode_wall_ms_total": level.timing_stats("llm_decode_wall_ms_total"),
scripts/stress_test_optimized_local.py:839:                "bicodec_decode_wall_ms": level.timing_stats("bicodec_decode_wall_ms"),
scripts/stress_test_optimized_local.py:840:                "bicodec_decode_gpu_ms": level.timing_stats("bicodec_decode_gpu_ms"),
scripts/stress_test_optimized_local.py:847:                "timeline_llm_first_batch_ms": level.timing_stats("timeline_llm_first_batch_ms"),
scripts/stress_test_optimized_local.py:848:                "timeline_llm_done_ms": level.timing_stats("timeline_llm_done_ms"),
scripts/stress_test_optimized_local.py:849:                "timeline_parse_done_ms": level.timing_stats("timeline_parse_done_ms"),
scripts/stress_test_optimized_local.py:850:                "timeline_bicodec_done_ms": level.timing_stats("timeline_bicodec_done_ms"),
scripts/stress_test_optimized_local.py:851:                "timeline_total_ms": level.timing_stats("timeline_total_ms"),
scripts/stress_test_optimized_local.py:859:                    "ttfb_ms": r.ttfb_ms,
scripts/stress_test_optimized_local.py:860:                    "server_ttfb_ms": r.server_ttfb_ms,
scripts/stress_test_optimized_local.py:902:        chunking=args.chunking,
scripts/stress_test_optimized_local.py:903:        max_tokens=args.max_tokens,
scripts/stress_test_optimized_local.py:918:            chunking=args.chunking,
scripts/stress_test_optimized_local.py:919:            max_tokens=args.max_tokens,
scripts/stress_test_optimized_local.py:943:    parser.add_argument("--chunking", action="store_true", help="Enable server-side chunking", default=True)
scripts/stress_test_optimized_local.py:944:    parser.add_argument("--no-chunking", action="store_false", dest="chunking", help="Disable chunking")
scripts/stress_test_optimized_local.py:946:    parser.add_argument("--max-tokens", type=int, default=128, help="Max tokens per request")
scripts/measure_ttfb_detailed.py:17:def measure_streaming_ttfb(text="Hello world, this is a test of streaming audio generation.", speaker="lipakshi"):
scripts/measure_ttfb_detailed.py:59:        # Read first chunk
scripts/measure_ttfb_detailed.py:60:        first_chunk = None
scripts/measure_ttfb_detailed.py:61:        chunk_count = 0
scripts/measure_ttfb_detailed.py:64:        for chunk in response.iter_content(chunk_size=4096):
scripts/measure_ttfb_detailed.py:65:            if chunk:
scripts/measure_ttfb_detailed.py:66:                chunk_count += 1
scripts/measure_ttfb_detailed.py:67:                total_bytes += len(chunk)
scripts/measure_ttfb_detailed.py:69:                if first_chunk is None:
scripts/measure_ttfb_detailed.py:70:                    first_chunk = chunk
scripts/measure_ttfb_detailed.py:71:                    t2_first_chunk = time.time() - t0_request_start
scripts/measure_ttfb_detailed.py:72:                    print(f"\n🎵 T2 = {t2_first_chunk:.3f}s - FIRST AUDIO CHUNK RECEIVED")
scripts/measure_ttfb_detailed.py:73:                    print(f"   └─ Size: {len(chunk)} bytes")
scripts/measure_ttfb_detailed.py:74:                    print(f"   └─ TTFB (True): {t2_first_chunk:.3f}s = {t2_first_chunk*1000:.0f}ms")
scripts/measure_ttfb_detailed.py:77:                    if chunk.startswith(b'RIFF'):
scripts/measure_ttfb_detailed.py:79:                        if len(chunk) > 44:
scripts/measure_ttfb_detailed.py:80:                            print(f"   └─ Audio data starts: {len(chunk)-44} bytes in first chunk")
scripts/measure_ttfb_detailed.py:86:        print(f"   └─ Total chunks: {chunk_count}")
scripts/measure_ttfb_detailed.py:93:        print(f"   ├─ First chunk received:  {t2_first_chunk*1000:>7.0f}ms  ← TRUE TTFB")
scripts/measure_ttfb_detailed.py:97:            'ttfb': t2_first_chunk,
scripts/measure_ttfb_detailed.py:99:            'chunks': chunk_count,
scripts/measure_ttfb_detailed.py:110:def measure_non_streaming_ttfb(text="Hello world, this is a test.", speaker="lipakshi"):
scripts/measure_ttfb_detailed.py:147:        server_ttfb = None
scripts/measure_ttfb_detailed.py:152:                    server_ttfb = float(value) / 1000  # Convert to seconds
scripts/measure_ttfb_detailed.py:163:        if server_ttfb:
scripts/measure_ttfb_detailed.py:164:            print(f"   ├─ Server reported (gen):    {server_ttfb*1000:>7.0f}ms  ← Generation time")
scripts/measure_ttfb_detailed.py:165:            network_time = t1_response_complete - server_ttfb
scripts/measure_ttfb_detailed.py:173:            'server_ttfb': server_ttfb,
scripts/measure_ttfb_detailed.py:198:    streaming_short = measure_streaming_ttfb(
scripts/measure_ttfb_detailed.py:207:    non_streaming_short = measure_non_streaming_ttfb(
scripts/measure_ttfb_detailed.py:217:    streaming_long = measure_streaming_ttfb(
scripts/measure_ttfb_detailed.py:228:        print(f"\n✅ Streaming (short):    TTFB = {streaming_short['ttfb']*1000:>6.0f}ms")
scripts/measure_ttfb_detailed.py:232:        print(f"✅ Streaming (long):     TTFB = {streaming_long['ttfb']*1000:>6.0f}ms")
scripts/measure_ttfb_detailed.py:235:    print(f"   • Streaming TTFB = Time to first audio chunk (TRUE TTFB)")
scripts/profile_pipeline.py:8:- vLLM decode (per-token generation)
scripts/profile_pipeline.py:10:- BiCodec audio decode (GPU)
scripts/profile_pipeline.py:19:    python scripts/profile_pipeline.py
scripts/profile_pipeline.py:21:    # Concurrency profiling (1, 5, 10, 20 parallel requests)
scripts/profile_pipeline.py:22:    python scripts/profile_pipeline.py --concurrency 1,5,10,20
scripts/profile_pipeline.py:24:    # Long text profiling
scripts/profile_pipeline.py:25:    python scripts/profile_pipeline.py --text long
scripts/profile_pipeline.py:27:    # Streaming profiling
scripts/profile_pipeline.py:28:    python scripts/profile_pipeline.py --stream
scripts/profile_pipeline.py:58:logger = logging.getLogger("profiler")
scripts/profile_pipeline.py:114:    """Complete timing profile for one TTS request."""
scripts/profile_pipeline.py:121:    # Metadata filled during profiling
scripts/profile_pipeline.py:123:    prompt_tokens: int = 0
scripts/profile_pipeline.py:124:    generated_tokens: int = 0
scripts/profile_pipeline.py:125:    semantic_tokens: int = 0
scripts/profile_pipeline.py:126:    global_tokens: int = 0
scripts/profile_pipeline.py:131:    ttfb_ms: float = 0.0
scripts/profile_pipeline.py:132:    chunks_emitted: int = 0
scripts/profile_pipeline.py:133:    token_generation_rates: List[float] = field(default_factory=list)  # tokens/sec samples
scripts/profile_pipeline.py:134:    decode_latencies: List[float] = field(default_factory=list)  # ms per BiCodec decode
scripts/profile_pipeline.py:148:            f"  Generated: {self.generated_tokens} tokens ({self.semantic_tokens} semantic, {self.global_tokens} global)",
scripts/profile_pipeline.py:152:            lines.append(f"  TTFB: {self.ttfb_ms:.1f}ms, Chunks: {self.chunks_emitted}")
scripts/profile_pipeline.py:153:        if self.generated_tokens > 0 and self.total_ms > 0:
scripts/profile_pipeline.py:154:            tps = self.generated_tokens / (self.total_ms / 1000)
scripts/profile_pipeline.py:182:        # BiCodec decode latencies
scripts/profile_pipeline.py:183:        if self.decode_latencies:
scripts/profile_pipeline.py:187:                f"    avg={statistics.mean(self.decode_latencies):.2f}  "
scripts/profile_pipeline.py:188:                f"min={min(self.decode_latencies):.2f}  "
scripts/profile_pipeline.py:189:                f"max={max(self.decode_latencies):.2f}  "
scripts/profile_pipeline.py:190:                f"calls={len(self.decode_latencies)}"
scripts/profile_pipeline.py:199:async def profile_non_streaming(
scripts/profile_pipeline.py:206:    max_tokens: int = 4096,
scripts/profile_pipeline.py:215:    profile = RequestProfile(
scripts/profile_pipeline.py:227:    profile.stage("1_prompt_build").record((time.perf_counter() - t0) * 1000)
scripts/profile_pipeline.py:228:    profile.prompt_tokens = len(runtime.model.tokenizer.encode(prompt, add_special_tokens=False))
scripts/profile_pipeline.py:236:        max_tokens=max_tokens,
scripts/profile_pipeline.py:239:        skip_special_tokens=False,
scripts/profile_pipeline.py:242:    profile.stage("2_sampling_config").record((time.perf_counter() - t0) * 1000)
scripts/profile_pipeline.py:265:            profile.stage("3a_vllm_prefill").record(prefill_ms)
scripts/profile_pipeline.py:269:        new_tokens = current_count - prev_token_count
scripts/profile_pipeline.py:270:        if new_tokens > 0 and iteration_count > 1:
scripts/profile_pipeline.py:271:            # Estimate instantaneous tokens/sec
scripts/profile_pipeline.py:272:            # (note: vLLM may batch multiple tokens per yield)
scripts/profile_pipeline.py:275:                profile.token_generation_rates.append(current_count / dt)
scripts/profile_pipeline.py:281:    decode_only_ms = total_gen_ms - (profile.stages.get("3a_vllm_prefill", StageTimer("")).total_ms)
scripts/profile_pipeline.py:282:    profile.stage("3b_vllm_decode").record(decode_only_ms)
scripts/profile_pipeline.py:283:    profile.stage("3_vllm_total").record(total_gen_ms)
scripts/profile_pipeline.py:284:    profile.generated_tokens = len(final_output.outputs[0].token_ids) if final_output else 0
scripts/profile_pipeline.py:287:        profile.total_ms = (time.perf_counter() - t_total_start) * 1000
scripts/profile_pipeline.py:288:        return profile
scripts/profile_pipeline.py:298:    profile.stage("4_token_extraction").record((time.perf_counter() - t0) * 1000)
scripts/profile_pipeline.py:299:    profile.semantic_tokens = len(semantic_ids)
scripts/profile_pipeline.py:300:    profile.global_tokens = len(global_ids)
scripts/profile_pipeline.py:303:        profile.total_ms = (time.perf_counter() - t_total_start) * 1000
scripts/profile_pipeline.py:304:        return profile
scripts/profile_pipeline.py:308:    runtime.bicodec_decoder.validate_tokens(semantic_ids, global_ids)
scripts/profile_pipeline.py:309:    profile.stage("5_token_validation").record((time.perf_counter() - t0) * 1000)
scripts/profile_pipeline.py:313:    audio_bytes = runtime.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
scripts/profile_pipeline.py:314:    decode_ms = (time.perf_counter() - t0) * 1000
scripts/profile_pipeline.py:315:    profile.stage("6_bicodec_decode").record(decode_ms)
scripts/profile_pipeline.py:316:    profile.decode_latencies.append(decode_ms)
scripts/profile_pipeline.py:319:        profile.total_ms = (time.perf_counter() - t_total_start) * 1000
scripts/profile_pipeline.py:320:        return profile
scripts/profile_pipeline.py:325:    profile.stage("7_wav_header").record((time.perf_counter() - t0) * 1000)
scripts/profile_pipeline.py:327:    profile.audio_bytes = len(wav_bytes)
scripts/profile_pipeline.py:328:    profile.audio_seconds = (len(audio_bytes)) / (AUDIO_SAMPLE_RATE * 2)  # int16 = 2 bytes/sample
scripts/profile_pipeline.py:329:    profile.total_ms = (time.perf_counter() - t_total_start) * 1000
scripts/profile_pipeline.py:331:    return profile
scripts/profile_pipeline.py:336:async def profile_streaming(
scripts/profile_pipeline.py:343:    max_tokens: int = 4096,
scripts/profile_pipeline.py:347:    """Profile a single streaming request with per-token and per-decode timing."""
scripts/profile_pipeline.py:353:    profile = RequestProfile(
scripts/profile_pipeline.py:365:    profile.stage("1_prompt_build").record((time.perf_counter() - t0) * 1000)
scripts/profile_pipeline.py:371:    profile.stage("2_parser_init").record((time.perf_counter() - t0) * 1000)
scripts/profile_pipeline.py:379:        max_tokens=max_tokens,
scripts/profile_pipeline.py:380:        min_tokens=DEFAULT_MIN_TOKENS,
scripts/profile_pipeline.py:385:    profile.stage("3_sampling_config").record((time.perf_counter() - t0) * 1000)
scripts/profile_pipeline.py:402:    t_first_decode = None
scripts/profile_pipeline.py:403:    last_decode_count = 0
scripts/profile_pipeline.py:405:    previous_chunk_tail: Optional[bytes] = None
scripts/profile_pipeline.py:430:            profile.stage("4a_vllm_prefill").record(prefill_ms)
scripts/profile_pipeline.py:439:        profile.stage("4b_token_parsing").record((time.perf_counter() - t_parse) * 1000)
scripts/profile_pipeline.py:445:            profile.stage("4c_global_token_gen").record(globals_ms)
scripts/profile_pipeline.py:447:        # === BiCodec Decode (when enough tokens) ===
scripts/profile_pipeline.py:450:            if sem_count >= MIN_SEMANTIC_FIRST and sem_count - last_decode_count >= DECODE_INTERVAL:
scripts/profile_pipeline.py:451:                last_decode_count = sem_count
scripts/profile_pipeline.py:453:                t_decode = time.perf_counter()
scripts/profile_pipeline.py:454:                audio_bytes = runtime.bicodec_decoder.decode_streaming(
scripts/profile_pipeline.py:460:                decode_ms = (time.perf_counter() - t_decode) * 1000
scripts/profile_pipeline.py:461:                profile.stage("5_bicodec_decode").record(decode_ms)
scripts/profile_pipeline.py:462:                profile.decode_latencies.append(decode_ms)
scripts/profile_pipeline.py:465:                    total_decoded_samples = len(audio_bytes) // 2
scripts/profile_pipeline.py:466:                    if total_decoded_samples > total_samples_emitted:
scripts/profile_pipeline.py:470:                        to_emit, previous_chunk_tail = crossfade_bytes_int16(
scripts/profile_pipeline.py:471:                            previous_chunk_tail, new_bytes,
scripts/profile_pipeline.py:474:                        profile.stage("6_crossfade").record((time.perf_counter() - t_cf) * 1000)
scripts/profile_pipeline.py:480:                            profile.chunks_emitted += 1
scripts/profile_pipeline.py:481:                            if t_first_decode is None:
scripts/profile_pipeline.py:482:                                t_first_decode = time.perf_counter()
scripts/profile_pipeline.py:483:                                profile.ttfb_ms = (t_first_decode - t_total_start) * 1000
scripts/profile_pipeline.py:486:    profile.stage("4_vllm_total").record((t_gen_end - t_gen_start) * 1000)
scripts/profile_pipeline.py:488:    # Final decode
scripts/profile_pipeline.py:489:    if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
scripts/profile_pipeline.py:490:        t_decode = time.perf_counter()
scripts/profile_pipeline.py:491:        audio_bytes = runtime.bicodec_decoder.decode_streaming(
scripts/profile_pipeline.py:496:        decode_ms = (time.perf_counter() - t_decode) * 1000
scripts/profile_pipeline.py:497:        profile.stage("5_bicodec_decode").record(decode_ms)
scripts/profile_pipeline.py:498:        profile.decode_latencies.append(decode_ms)
scripts/profile_pipeline.py:501:            total_decoded_samples = len(audio_bytes) // 2
scripts/profile_pipeline.py:502:            if total_decoded_samples > total_samples_emitted:
scripts/profile_pipeline.py:504:                to_emit, previous_chunk_tail = crossfade_bytes_int16(
scripts/profile_pipeline.py:505:                    previous_chunk_tail, new_bytes, sample_rate_hz=16000, crossfade_ms=50,
scripts/profile_pipeline.py:509:                    profile.chunks_emitted += 1
scripts/profile_pipeline.py:511:    if previous_chunk_tail:
scripts/profile_pipeline.py:512:        total_samples_emitted += len(previous_chunk_tail) // 2
scripts/profile_pipeline.py:513:        profile.chunks_emitted += 1
scripts/profile_pipeline.py:515:    profile.generated_tokens = processed_token_count
scripts/profile_pipeline.py:516:    profile.semantic_tokens = len(semantic_buffer)
scripts/profile_pipeline.py:517:    profile.global_tokens = len(global_buffer)
scripts/profile_pipeline.py:518:    profile.audio_seconds = total_samples_emitted / 16000
scripts/profile_pipeline.py:519:    profile.audio_bytes = total_samples_emitted * 2
scripts/profile_pipeline.py:520:    profile.total_ms = (time.perf_counter() - t_total_start) * 1000
scripts/profile_pipeline.py:523:    # Skip first (prefill) and compute tok/s for decode phase
scripts/profile_pipeline.py:525:        decode_times = inter_token_times[1:]  # Skip prefill
scripts/profile_pipeline.py:526:        for dt in decode_times:
scripts/profile_pipeline.py:528:                profile.token_generation_rates.append(1000.0 / dt)  # ms → tok/s
scripts/profile_pipeline.py:533:        decode_times = inter_token_times[1:] if len(inter_token_times) > 1 else inter_token_times
scripts/profile_pipeline.py:534:        if decode_times:
scripts/profile_pipeline.py:535:            profile.stage("4d_inter_token_avg").record(statistics.mean(decode_times))
scripts/profile_pipeline.py:536:            profile.stage("4e_inter_token_p99").record(
scripts/profile_pipeline.py:537:                sorted(decode_times)[int(len(decode_times) * 0.99)]
scripts/profile_pipeline.py:540:    return profile
scripts/profile_pipeline.py:545:async def profile_concurrent(
scripts/profile_pipeline.py:559:            return await profile_streaming(runtime, text, spk)
scripts/profile_pipeline.py:561:            return await profile_non_streaming(runtime, text, spk)
scripts/profile_pipeline.py:563:    profiles = await asyncio.gather(*[run_one(i) for i in range(num_requests)])
scripts/profile_pipeline.py:564:    return list(profiles)
scripts/profile_pipeline.py:567:def print_concurrency_summary(profiles: List[RequestProfile], concurrency: int, stream: bool):
scripts/profile_pipeline.py:568:    """Print aggregated summary for concurrent profiles."""
scripts/profile_pipeline.py:572:    totals = [p.total_ms for p in profiles]
scripts/profile_pipeline.py:576:    print(f"  CONCURRENCY={concurrency} ({mode}), {len(profiles)} requests")
scripts/profile_pipeline.py:579:    print(f"  Throughput: {len(profiles) / (wall_time / 1000):.2f} req/s")
scripts/profile_pipeline.py:585:        ttfbs = [p.ttfb_ms for p in profiles if p.ttfb_ms > 0]
scripts/profile_pipeline.py:586:        if ttfbs:
scripts/profile_pipeline.py:587:            print(f"  TTFB: avg={statistics.mean(ttfbs):.0f}ms  "
scripts/profile_pipeline.py:588:                  f"p50={sorted(ttfbs)[len(ttfbs)//2]:.0f}ms  "
scripts/profile_pipeline.py:589:                  f"p95={sorted(ttfbs)[int(len(ttfbs)*0.95)]:.0f}ms")
scripts/profile_pipeline.py:593:    for p in profiles:
scripts/profile_pipeline.py:599:    print(f"\n  STAGE BREAKDOWN (avg across {len(profiles)} requests):")
scripts/profile_pipeline.py:609:    # BiCodec decode contention
scripts/profile_pipeline.py:610:    all_decode_lats = []
scripts/profile_pipeline.py:611:    for p in profiles:
scripts/profile_pipeline.py:612:        all_decode_lats.extend(p.decode_latencies)
scripts/profile_pipeline.py:613:    if all_decode_lats:
scripts/profile_pipeline.py:614:        print(f"\n  BICODEC DECODE (all calls across {len(profiles)} requests):")
scripts/profile_pipeline.py:615:        print(f"    total calls: {len(all_decode_lats)}")
scripts/profile_pipeline.py:616:        print(f"    avg={statistics.mean(all_decode_lats):.2f}ms  "
scripts/profile_pipeline.py:617:              f"p50={sorted(all_decode_lats)[len(all_decode_lats)//2]:.2f}ms  "
scripts/profile_pipeline.py:618:              f"p99={sorted(all_decode_lats)[int(len(all_decode_lats)*0.99)]:.2f}ms  "
scripts/profile_pipeline.py:619:              f"max={max(all_decode_lats):.2f}ms")
scripts/profile_pipeline.py:623:    for p in profiles:
scripts/profile_pipeline.py:650:    logger.info("Initializing TTS runtime for profiling...")
scripts/profile_pipeline.py:684:            p = await profile_non_streaming(runtime, text, "lipakshi")
scripts/profile_pipeline.py:690:            p = await profile_streaming(runtime, text, "lipakshi")
scripts/profile_pipeline.py:703:            profiles = await profile_concurrent(runtime, conc, text, stream=stream_mode)
scripts/profile_pipeline.py:704:            print_concurrency_summary(profiles, conc, stream_mode)
scripts/profile_pipeline.py:717:    parser.add_argument("--concurrency-only", action="store_true", help="Skip single-request profiles")


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Reviewing existing metrics and vLLM setup**


I confirmed we already have timeline/header plumbing in the codebase, so I’m now reading the runtime and streaming internals to target the biggest decode/scheduling/queue bottleneck rather than adding redundant instrumentation.


cd /home/ubuntu/veenaModal && sed -n '1,260p' veena3modal/services/tts_runtime.py


cd /home/ubuntu/veenaModal && sed -n '1,320p' veena3modal/core/streaming_pipeline.py


cd /home/ubuntu/veenaModal && sed -n '1,340p' veena3modal/core/pipeline.py


Output:
Chunk ID: a258ed
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 2623
Output:
"""
TTS runtime wrapper (container-scoped singleton).

In Modal, each container should load the model once and then handle many requests.
This wrapper owns:
- SparkTTSModel (vLLM engine)
- IndicPromptBuilder
- BiCodecDecoder
- SparkTTSPipeline + Veena3SlidingWindowPipeline
- optional SuperResolutionService

Imports framework-agnostic inference + processing code from `veena3modal/core`,
`veena3modal/processing`, and `veena3modal/audio`.
"""

from __future__ import annotations

import os
import sys
import time
import logging
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

logger = logging.getLogger(__name__)

@dataclass
class TTSRuntime:
    """
    Holds long-lived, per-container inference objects.

    Thread Safety:
        This class is designed for async concurrency within a single container.
        The vLLM engine handles internal batching and scheduling.
        Do NOT share across processes without proper synchronization.
    """
    model: Any = None
    pipeline: Any = None
    streaming_pipeline: Any = None
    long_text_processor: Any = None
    prompt_builder: Any = None
    bicodec_decoder: Any = None
    sr_service: Optional[Any] = None

    model_version: str = "not_loaded"
    is_loaded: bool = False
    load_time_ms: float = 0.0

    # Configuration
    model_path: str = ""
    bicodec_path: str = ""
    sr_checkpoint_dir: Optional[str] = None
    device: str = "cuda"

    # OPTIMIZATION: Pre-computed global tokens per speaker (eliminates ~110ms pre-roll in streaming)
    # Maps speaker_name -> list of 32 global token IDs captured at startup
    speaker_global_cache: Dict[str, List[int]] = field(default_factory=dict)

# Module-level singleton (per-container)
_runtime: Optional[TTSRuntime] = None

def get_runtime() -> Optional[TTSRuntime]:
    """Get the current TTS runtime singleton."""
    return _runtime

def is_initialized() -> bool:
    """Check if the runtime is initialized and ready."""
    return _runtime is not None and _runtime.is_loaded

def initialize_runtime(
    model_path: Optional[str] = None,
    bicodec_path: Optional[str] = None,
    sr_checkpoint_dir: Optional[str] = None,
    device: str = "cuda",
    hf_token: Optional[str] = None,
    gpu_memory_utilization: float = 0.25,
    enable_sr: bool = False,
    num_engines: int = 1,
    max_num_batched_tokens: Optional[int] = None,
    max_num_seqs: Optional[int] = None,
    enable_chunked_prefill: Optional[bool] = None,
    enable_prefix_caching: Optional[bool] = None,
    disable_log_stats: Optional[bool] = None,
    enforce_eager: Optional[bool] = None,
    precompute_speaker_globals: Optional[bool] = None,
) -> TTSRuntime:
    """
    Initialize the TTS runtime with all components.

    This should be called once per container (e.g., in Modal's @modal.enter).

    Args:
        model_path: Path to Spark TTS model (env: SPARK_TTS_MODEL_PATH)
        bicodec_path: Path to BiCodec model (env: BICODEC_MODEL_PATH, defaults to model_path)
        sr_checkpoint_dir: Path to super-resolution checkpoints (env: AP_BWE_CHECKPOINT_DIR)
        device: Device for inference (cuda/cpu)
        hf_token: HuggingFace token for private models (env: HF_TOKEN)
        gpu_memory_utilization: vLLM GPU memory fraction (default: 0.25)
        enable_sr: Enable super-resolution service
        num_engines: Number of vLLM engine instances (default: 1, set to 2-3 for Tier 3 optimization)
        max_num_batched_tokens: Optional vLLM scheduler cap override.
        max_num_seqs: Optional vLLM concurrent sequence cap override.
        enable_chunked_prefill: Optional vLLM chunked prefill toggle.
        enable_prefix_caching: Optional vLLM prefix caching toggle.
        disable_log_stats: Optional vLLM internal stats log toggle.
        enforce_eager: Optional vLLM eager-mode toggle (disables CUDA graphs when true).
        precompute_speaker_globals: Whether to warm speaker global tokens at startup.
            If None, reads PRECOMPUTE_SPEAKER_GLOBALS env var (default: false).

    Returns:
        Initialized TTSRuntime instance

    Raises:
        RuntimeError: If initialization fails
    """
    global _runtime

    start_time = time.time()

    # Resolve paths from env vars if not provided
    model_path = model_path or os.environ.get(
        'SPARK_TTS_MODEL_PATH',
        os.environ.get('MODEL_PATH', '/models/spark_tts_4speaker')
    )
    bicodec_path = bicodec_path or os.environ.get(
        'BICODEC_MODEL_PATH',
        model_path  # BiCodec is usually in the same directory
    )
    sr_checkpoint_dir = sr_checkpoint_dir or os.environ.get('AP_BWE_CHECKPOINT_DIR')
    hf_token = hf_token or os.environ.get('HF_TOKEN') or os.environ.get('HUGGING_FACE_HUB_TOKEN')
    if precompute_speaker_globals is None:
        raw_flag = os.environ.get("PRECOMPUTE_SPEAKER_GLOBALS", "false").strip().lower()
        precompute_speaker_globals = raw_flag in {"1", "true", "yes", "on"}

    logger.info(f"Initializing TTS runtime...")
    logger.info(f"  Model path: {model_path}")
    logger.info(f"  BiCodec path: {bicodec_path}")
    logger.info(f"  SR enabled: {enable_sr}, path: {sr_checkpoint_dir}")

    try:
        # Import inference components (vendored into veena3modal for Modal deployment)
        from veena3modal.core.model_loader import SparkTTSModel
        from veena3modal.core.pipeline import SparkTTSPipeline
        from veena3modal.core.bicodec_decoder import BiCodecDecoder
        from veena3modal.processing.prompt_builder import IndicPromptBuilder
        from veena3modal.processing.long_text_processor import LongTextProcessor

        engine_kwargs: Dict[str, Any] = {}
        if isinstance(max_num_batched_tokens, int) and max_num_batched_tokens > 0:
            engine_kwargs["max_num_batched_tokens"] = max_num_batched_tokens
        if isinstance(max_num_seqs, int) and max_num_seqs > 0:
            engine_kwargs["max_num_seqs"] = max_num_seqs
        if enable_chunked_prefill is not None:
            engine_kwargs["enable_chunked_prefill"] = bool(enable_chunked_prefill)
        if enable_prefix_caching is not None:
            engine_kwargs["enable_prefix_caching"] = bool(enable_prefix_caching)
        if disable_log_stats is not None:
            engine_kwargs["disable_log_stats"] = bool(disable_log_stats)
        if enforce_eager is not None:
            engine_kwargs["enforce_eager"] = bool(enforce_eager)

        # Load SparkTTS model with vLLM engine
        if num_engines > 1:
            # TIER 3 OPTIMIZATION: Multiple vLLM engines on same GPU
            # Each engine gets gpu_memory_utilization / num_engines to share GPU fairly.
            # E.g., 3 engines at 0.08 each = 0.24 total, ~8GB per engine, ~24GB total.
            from veena3modal.core.multi_engine import create_multi_engine_model
            per_engine_mem = gpu_memory_utilization / num_engines
            logger.info(f"Loading {num_engines} vLLM engines ({per_engine_mem:.2f} GPU mem each)...")
            model = create_multi_engine_model(
                model_path=model_path,
                num_engines=num_engines,
                hf_token=hf_token,
                gpu_memory_per_engine=per_engine_mem,
                **engine_kwargs,
            )
        else:
            logger.info("Loading SparkTTS model with vLLM...")
            model = SparkTTSModel(
                model_path=model_path,
                hf_token=hf_token,
                gpu_memory_utilization=gpu_memory_utilization,
                **engine_kwargs,
            )

        # Initialize prompt builder
        logger.info("Initializing prompt builder...")
        prompt_builder = IndicPromptBuilder(
            tokenizer=model.tokenizer,
            model=model,
        )

        # Initialize BiCodec decoder
        logger.info("Initializing BiCodec decoder...")
        bicodec_decoder = BiCodecDecoder(
            device=device,
            model_path=bicodec_path,
        )

        # Initialize pipeline
        logger.info("Initializing TTS pipeline...")
        pipeline = SparkTTSPipeline(
            model=model,
            prompt_builder=prompt_builder,
            bicodec_decoder=bicodec_decoder,
        )

        # Initialize streaming pipeline (for M4)
        streaming_pipeline = None
        try:
            from veena3modal.core.streaming_pipeline import Veena3SlidingWindowPipeline
            logger.info("Initializing streaming pipeline...")
            # NOTE: Parameter is named 'snac_decoder' for legacy reasons, but works with BiCodecDecoder
            streaming_pipeline = Veena3SlidingWindowPipeline(
                model=model,
                prompt_builder=prompt_builder,
                snac_decoder=bicodec_decoder,  # BiCodecDecoder is interface-compatible
            )
        except ImportError as e:
            logger.warning(f"Streaming pipeline not available: {e}")

        # Initialize long text processor once (avoid per-request construction/logging)
        long_text_processor = LongTextProcessor(
            pipeline=pipeline,
            streaming_pipeline=streaming_pipeline,
        )

        # Initialize super-resolution (optional)
        sr_service = None
        if enable_sr and sr_checkpoint_dir:
            try:
                from veena3modal.core.super_resolution import SuperResolutionService
                logger.info(f"Initializing super-resolution from {sr_checkpoint_dir}...")
                sr_service = SuperResolutionService(checkpoint_dir=sr_checkpoint_dir)
                # Load the model explicitly
                if sr_service.load_model(device=device):
                    logger.info("✅ Super-resolution model loaded successfully")
                else:
                    logger.warning("Super-resolution model failed to load")
                    sr_service = None
            except Exception as e:
                logger.warning(f"Super-resolution not available: {e}")

        # Determine model version
        model_version = os.path.basename(model_path.rstrip('/'))
        if not model_version:
            model_version = "spark-tts"

        load_time = (time.time() - start_time) * 1000

        # Create runtime
        _runtime = TTSRuntime(
            model=model,
            pipeline=pipeline,
            streaming_pipeline=streaming_pipeline,


Output:
Chunk ID: aeb861
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 3205
Output:
"""
Veena3 Streaming Pipeline - Sliding Window Approach

Implements streaming for both SNAC and BiCodec tokens.

For BiCodec (Spark TTS):
1. Generate tokens from vLLM
2. Parse NEW token IDs incrementally (O(1) per token via cache)
3. Buffer semantic + global tokens
4. Apply sliding window (every N token pairs)
5. Decode and stream audio chunks

OPTIMIZATION (Dec 2025):
- Replaced O(n²) pattern (decode-all + regex-all per iteration)
- Now uses incremental token parsing with BiCodecTokenParser
- ~10x CPU reduction in streaming hot loop

For SNAC (legacy):
1. Filter SNAC token IDs directly
2. Apply sliding window
3. Decode and stream
"""

import asyncio
import logging
import os
import re
from typing import AsyncGenerator, Optional, List
from vllm import SamplingParams
from vllm.sampling_params import RequestOutputKind
from veena3modal.audio.crossfade import crossfade_bytes_int16
from veena3modal.core.token_utils import BiCodecTokenParser

from veena3modal.core.constants import (
    CODE_END_TOKEN_ID,
    CODE_START_TOKEN_ID,
    SNAC_MIN_ID,
    SNAC_MAX_ID,
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_MIN_TOKENS,
    DEFAULT_REPETITION_PENALTY,
    DEFAULT_SEED,
)

logger = logging.getLogger(__name__)
_STREAM_OUTPUT_KIND_RAW = os.environ.get("VEENA3_STREAM_OUTPUT_KIND", "delta").strip().lower()
_STREAM_USE_DELTA_OUTPUT = _STREAM_OUTPUT_KIND_RAW in {"delta", "d", "1", "true", "yes", "on"}

class Veena3SlidingWindowPipeline:
    """
    Streaming TTS pipeline using sliding window approach.

    This eliminates choppy audio and popping artifacts by:
    - Decoding overlapping 28-token windows (4 frames)
    - Keeping only the middle 2048 samples from each decode
    - Creating natural continuity between chunks

    Based on the official Canopy Labs implementation.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        snac_decoder,
    ):
        """
        Initialize sliding window streaming pipeline.

        Args:
            model: Veena3Model instance
            prompt_builder: Veena3PromptBuilder instance
            snac_decoder: SNACDecoder instance (with batching enabled)
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.snac_decoder = snac_decoder

        # OPTIMIZATION: Pre-warm BiCodecTokenParser once at init, not per-request
        # Saves ~123ms per streaming TTFB by avoiding repeated 166K-entry vocab iteration
        tokenizer = getattr(model, "tokenizer", None)
        if tokenizer is None:
            tokenizer = getattr(model.engine, "tokenizer", None)
        self.token_parser = BiCodecTokenParser(tokenizer) if tokenizer else None

        print(f"🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)")

    async def generate_speech_stream(
        self,
        description: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech audio with sliding window streaming.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            description: Character/voice description
            text: Text to synthesize (with optional <emotion> tags)
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops

        Yields:
            Audio bytes (int16 PCM, 24kHz mono)
        """
        logger.debug("sliding-window streaming generation started")
        logger.debug("description_prefix=%s", description[:80])
        logger.debug("text_len=%d", len(text))

        # Build prompt
        prompt = self.prompt_builder.build_prefix(description, text)
        logger.debug("prompt built chars=%d", len(prompt))

        # Configure sampling
        sampling_params = SamplingParams(
            temperature=temperature,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop_token_ids=[CODE_END_TOKEN_ID],  # Only audio EOS
            seed=seed,  # None = random, int = reproducible
        )

        logger.debug("sampling temp=%s top_p=%s sliding_window=28", temperature, top_p)

        # Token buffer - keeps ALL tokens (not chunked)
        token_buffer = []
        total_tokens_generated = 0
        total_audio_chunks = 0

        # Generate tokens with vLLM (streaming)
        print(f"🔮 Starting token generation...")

        # Generate unique request ID for concurrent streaming support
        import uuid
        import time
        request_id = f"slide-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Stream tokens with sliding window
        async for request_output in results_generator:
            # Extract generated token IDs
            generated_ids = request_output.outputs[0].token_ids

            # Process only new tokens
            new_tokens = generated_ids[total_tokens_generated:]
            total_tokens_generated = len(generated_ids)

            # Filter and buffer SNAC tokens
            for token_id in new_tokens:
                if SNAC_MIN_ID <= token_id <= SNAC_MAX_ID:
                    token_buffer.append(token_id)

                    # Process every 7 tokens (1 frame) when we have enough for sliding window
                    # Official approach: once count > 27, take last 28 tokens
                    if len(token_buffer) % 7 == 0 and len(token_buffer) > 27:
                        # Sliding window: take last 28 tokens (4 frames)
                        window_tokens = token_buffer[-28:]

                        # Decode with sliding window mode (returns middle 2048 samples only)
                        if self.snac_decoder.enable_batching:
                            audio_bytes = await self.snac_decoder.decode_single_async(
                                window_tokens,
                                trim_warmup=False,  # Sliding window handles trimming
                                use_sliding_window=True  # CRITICAL: Use sliding window mode
                            )
                        else:
                            audio_bytes = self.snac_decoder.decode_to_bytes(
                                window_tokens,
                                trim_warmup=False,
                                use_sliding_window=True
                            )

                        if audio_bytes:
                            total_audio_chunks += 1
                            if total_audio_chunks == 1:
                                logger.debug("first sliding-window chunk decoded bytes=%d", len(audio_bytes))
                            yield audio_bytes

        # Note: No final chunk processing needed - sliding window handles all tokens
        # as they come in (every 7 tokens after the first 28)

        logger.debug(
            "sliding-window complete tokens=%d chunks=%d",
            total_tokens_generated,
            total_audio_chunks,
        )

    def _extract_bicodec_tokens_from_text(self, text: str) -> tuple[List[int], List[int]]:
        """
        Extract BiCodec semantic and global tokens from generated text.

        Args:
            text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
        """
        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", text)
        semantic_ids = [int(t) for t in semantic_matches]

        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", text)
        global_ids = [int(t) for t in global_matches]

        return semantic_ids, global_ids

    async def generate_speech_stream_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,  # Added for Spark TTS parity with non-streaming
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech audio with sliding window streaming for Indic model.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "<laugh> Hello there!"
                - "नमस्ते! <excited> आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops
            seed: Random seed for reproducibility

        Yields:
            Audio bytes (int16 PCM, 16kHz mono - BiCodec)
        """
        import time
        t_start = time.time()

        # Build prompt using Indic prompt builder
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling (matching non-streaming pipeline parameters)
        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID (128258)
        # which doesn't exist in the Spark vocab and would cause generation to run to max_tokens
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
            skip_special_tokens=False,  # Keep BiCodec tokens in output
            seed=seed,
            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
        )

        # BiCodec token buffers (separate for semantic and global)
        semantic_buffer = []
        global_buffer = []
        processed_token_count = 0  # Used for cumulative mode; ignored for delta mode.
        total_audio_chunks = 0

        # BiCodec streaming configuration
        EXPECTED_GLOBAL_COUNT = 32  # BiCodec uses 32 fixed global tokens (per paper)
        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8, -120ms TTFB)
        # OPTIMIZATION: Increased from 24 to 48 to reduce O(n) re-decode calls by 2x.
        # At 50 TPS, 48 tokens = ~960ms audio per chunk. Cuts total decode calls roughly in half.
        DECODE_INTERVAL = 48
        CROSSFADE_MS = 50
        # OPTIMIZATION: Windowed decode - only decode last WINDOW_SIZE tokens instead of ALL.
        # BiCodec WaveGenerator has finite receptive field (~64 tokens based on kernel sizes).
        # Window must be >= receptive_field + DECODE_INTERVAL to avoid boundary artifacts.
        WINDOW_SIZE = 128  # 128 tokens covers receptive field + margin for crossfade

        # Generate tokens with vLLM (streaming)
        import uuid
        request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"

        # Reuse pre-warmed token parser (singleton, created once in __init__)
        token_parser = self.token_parser

        t_gen_start = time.time()
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Track timing
        t_first_token = None
        t_first_chunk_ready = None
        t_first_chunk_decoded = None
        iteration_count = 0
        last_decode_count = 0

        # Track audio samples EMITTED to the user (not just decoded)
        # BiCodec produces ~320 samples per semantic token at 16kHz
        # CRITICAL: This tracks samples that have been YIELDED, not held in tail


Output:
Chunk ID: b06ec2
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 3003
Output:
"""
Spark TTS Generation Pipeline

End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
"""

import logging
import os
import re
import time
import uuid
from dataclasses import asdict, is_dataclass
from typing import Optional, List, Dict, Any, Tuple
from vllm import SamplingParams
from vllm.sampling_params import RequestOutputKind
import torch

logger = logging.getLogger(__name__)
_PERF_STAGE_LOG_ENABLED = os.environ.get("VEENA3_PERF_STAGE_LOG", "").strip().lower() in {
    "1",
    "true",
    "yes",
    "on",
}
_PERF_GPU_TIMING_ENABLED = os.environ.get("VEENA3_PERF_GPU_TIMING", "").strip().lower() in {
    "1",
    "true",
    "yes",
    "on",
}
_ASYNC_BICODEC_DECODE_ENABLED = os.environ.get("VEENA3_ASYNC_BICODEC_DECODE", "").strip().lower() in {
    "1",
    "true",
    "yes",
    "on",
}
_NON_STREAM_FINAL_ONLY_ENABLED = os.environ.get("VEENA3_NON_STREAM_FINAL_ONLY", "").strip().lower() in {
    "1",
    "true",
    "yes",
    "on",
}

def _log_perf_stage(event: str, extra: Dict[str, Any]) -> None:
    """Emit hot-path perf logs only when explicitly enabled or DEBUG logging is active."""
    if _PERF_STAGE_LOG_ENABLED:
        logger.info(event, extra=extra)
    elif logger.isEnabledFor(logging.DEBUG):
        logger.debug(event, extra=extra)

from veena3modal.core.constants import (
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_SEED,
    AUDIO_SAMPLE_RATE,
)

class SparkTTSPipeline:
    """
    End-to-end TTS pipeline for Spark TTS with BiCodec.

    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        bicodec_decoder,
    ):
        """
        Initialize pipeline.

        Args:
            model: SparkTTS Model instance
            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
            bicodec_decoder: BiCodecDecoder instance
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.bicodec_decoder = bicodec_decoder

        print(f"🚀 SparkTTSPipeline initialized")

    @staticmethod
    def _p50(values: List[float]) -> float:
        """Compute a stable p50 for small lists."""
        if not values:
            return 0.0
        ordered = sorted(values)
        return ordered[(len(ordered) - 1) // 2]

    @staticmethod
    def _to_number(value: Any) -> Optional[float]:
        """Best-effort conversion of scalar values to float."""
        if value is None:
            return None
        if isinstance(value, (int, float)):
            return float(value)
        try:
            return float(value)
        except (TypeError, ValueError):
            return None

    def _extract_request_metrics(self, raw_metrics: Any) -> Dict[str, float]:
        """
        Convert vLLM request_output.metrics into a numeric dictionary.

        vLLM may return a dataclass-like object or a plain dict depending on version.
        """
        if raw_metrics is None:
            return {}

        numeric: Dict[str, float] = {}
        items: List[Tuple[str, Any]] = []

        if isinstance(raw_metrics, dict):
            items.extend(raw_metrics.items())
        else:
            # Modern vLLM versions expose RequestStateStats dataclasses with
            # fields like queued_ts/scheduled_ts/first_token_ts/last_token_ts.
            if is_dataclass(raw_metrics):
                try:
                    items.extend(asdict(raw_metrics).items())
                except Exception:
                    pass
            if hasattr(raw_metrics, "__dict__"):
                try:
                    items.extend(dict(raw_metrics.__dict__).items())
                except Exception:
                    pass

            # Fallback: scan public scalar attributes.
            for name in dir(raw_metrics):
                if name.startswith("_"):
                    continue
                try:
                    value = getattr(raw_metrics, name)
                except Exception:
                    continue
                if callable(value):
                    continue
                items.append((name, value))

        seen: set[str] = set()
        for key, value in items:
            key_s = str(key)
            if key_s in seen:
                continue
            seen.add(key_s)
            parsed = self._to_number(value)
            if parsed is not None:
                numeric[key_s] = parsed
        return numeric

    def _apply_stats(self, perf: Dict[str, Any], prefix: str, values: List[float]) -> None:
        """Populate total/min/max/p50 stats for a list of ms samples."""
        if not values:
            return
        perf[f"{prefix}_total"] = float(sum(values))
        perf[f"{prefix}_min"] = float(min(values))
        perf[f"{prefix}_max"] = float(max(values))
        perf[f"{prefix}_p50"] = float(self._p50(values))

    async def generate_speech_profiled(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
    ) -> Tuple[Optional[bytes], Dict[str, Any]]:
        """
        Generate speech with detailed per-request timing metrics.

        Returns:
            (wav_bytes, perf_dict)
        """
        from veena3modal.audio.utils import add_wav_header

        perf: Dict[str, Any] = {
            "llm_prompt_token_total": 0,
            "llm_token_total": 0,
            "semantic_token_total": 0,
            "global_token_total": 0,
            "llm_batch_count": 0,
            "llm_parse_ms": 0.0,
            "llm_decode_calls": 0,
            "bicodec_decode_wall_ms": 0.0,
            "bicodec_decode_gpu_ms": 0.0,
            "bicodec_decode_cpu_ms": 0.0,
            "wav_pack_ms": 0.0,
        }

        t_request_start = time.perf_counter()
        timeline_marks: List[Dict[str, float]] = [{"stage": "request_start", "t_ms": 0.0, "dt_ms": 0.0}]
        last_mark_ms = 0.0
        perf["timeline_markers"] = timeline_marks
        perf["timeline_request_start_ms"] = 0.0

        def _mark(stage: str) -> float:
            nonlocal last_mark_ms
            now_ms = (time.perf_counter() - t_request_start) * 1000
            dt_ms = now_ms - last_mark_ms
            last_mark_ms = now_ms
            timeline_marks.append(
                {
                    "stage": stage,
                    "t_ms": float(now_ms),
                    "dt_ms": float(dt_ms),
                }
            )
            perf[f"timeline_{stage}_ms"] = float(now_ms)
            perf[f"timeline_{stage}_delta_ms"] = float(dt_ms)
            return float(now_ms)

        def _finalize_timeline() -> None:
            if not timeline_marks or timeline_marks[-1].get("stage") != "request_done":
                _mark("request_done")

            # Keep a compact, ordered timeline payload for downstream benchmarking.
            perf["timeline_markers"] = list(timeline_marks)
            perf["timeline_total_ms"] = float(perf.get("timeline_request_done_ms", 0.0))

            first_batch_ms = perf.get("timeline_llm_first_batch_ms")
            llm_done_ms = perf.get("timeline_llm_done_ms")
            parse_done_ms = perf.get("timeline_parse_done_ms")
            bicodec_done_ms = perf.get("timeline_bicodec_done_ms")
            request_done_ms = perf.get("timeline_request_done_ms")

            if isinstance(first_batch_ms, (int, float)):
                perf["timeline_to_first_batch_ms"] = float(first_batch_ms)
            if isinstance(first_batch_ms, (int, float)) and isinstance(llm_done_ms, (int, float)) and llm_done_ms >= first_batch_ms:
                perf["timeline_first_batch_to_llm_done_ms"] = float(llm_done_ms - first_batch_ms)
            if isinstance(llm_done_ms, (int, float)) and isinstance(request_done_ms, (int, float)) and request_done_ms >= llm_done_ms:
                perf["timeline_post_llm_ms"] = float(request_done_ms - llm_done_ms)
            if isinstance(parse_done_ms, (int, float)) and isinstance(bicodec_done_ms, (int, float)) and bicodec_done_ms >= parse_done_ms:
                perf["timeline_parse_to_bicodec_done_ms"] = float(bicodec_done_ms - parse_done_ms)
            if isinstance(bicodec_done_ms, (int, float)) and isinstance(request_done_ms, (int, float)) and request_done_ms >= bicodec_done_ms:
                perf["timeline_bicodec_to_request_done_ms"] = float(request_done_ms - bicodec_done_ms)

        # Stage 1: prompt build
        t0 = time.perf_counter()
        prompt = self.prompt_builder.build_prefix(speaker, text)
        perf["prompt_build_ms"] = (time.perf_counter() - t0) * 1000
        _mark("prompt_ready")

        # Prompt token count (for TPS normalization)
        try:
            perf["llm_prompt_token_total"] = len(
                self.model.tokenizer.encode(prompt, add_special_tokens=False)
            )
        except Exception:
            perf["llm_prompt_token_total"] = 0

        # Stage 2: sampling params
        t0 = time.perf_counter()
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
            output_kind=(RequestOutputKind.FINAL_ONLY if _NON_STREAM_FINAL_ONLY_ENABLED else RequestOutputKind.CUMULATIVE),
        )
        perf["sampling_params_ms"] = (time.perf_counter() - t0) * 1000
        _mark("sampling_ready")

        request_id = f"req-{uuid.uuid4().hex[:12]}"
        _log_perf_stage(
            "perf_non_stream_start",
            extra={
                "request_id": request_id,
                "speaker": speaker,
                "text_length": len(text),
                "max_tokens": max_tokens,
            },
        )
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        final_output = None
        final_request_metrics: Dict[str, float] = {}

        llm_batch_wall_ms: List[float] = []
        llm_batch_gpu_ms: List[float] = []
        llm_decode_wall_ms: List[float] = []
        llm_decode_gpu_ms: List[float] = []
        tokens_per_batch: List[float] = []

        prev_generated_count = 0
        prev_gpu_cumulative_s: Optional[float] = None

        t_gen_start = time.perf_counter()
        t_last_yield = t_gen_start
        async for request_output in results_generator:
            now = time.perf_counter()
            batch_wall = (now - t_last_yield) * 1000
            t_last_yield = now

            outputs = request_output.outputs or []
            if not outputs:
                final_output = request_output
                continue

            output = outputs[0]
            generated_count = len(output.token_ids)
            new_tokens = max(0, generated_count - prev_generated_count)
            prev_generated_count = generated_count

            raw_req_metrics = self._extract_request_metrics(getattr(request_output, "metrics", None))
            if raw_req_metrics:
                final_request_metrics = raw_req_metrics

            if new_tokens <= 0:
                final_output = request_output
                continue

            perf["llm_batch_count"] += 1
            if perf["llm_batch_count"] == 1:
                first_batch_timeline_ms = _mark("llm_first_batch")
                _log_perf_stage(
                    "perf_first_llm_batch",
                    extra={
                        "request_id": request_id,


Summary: auto


**Extending streaming pipeline context**


cd /home/ubuntu/veenaModal && sed -n '320,760p' veena3modal/core/streaming_pipeline.py


cd /home/ubuntu/veenaModal && sed -n '760,1240p' veena3modal/core/streaming_pipeline.py


cd /home/ubuntu/veenaModal && sed -n '260,700p' veena3modal/services/tts_runtime.py