Output:
Chunk ID: 8cf83f
Wall time: 0.0518 seconds
Process exited with code 0
Original token count: 247003
Output:
Total output lines: 2

keys: ['config', 'levels']
config: {'levels': [{'concurrency': 32, 'total_requests': 32, 'success_rate': 1.0, 'throughput_rps': 15.842462949717238, 'wall_time_s': 2.0198879493400455, 'latency_ms': {'avg': 1833.6132035183255, 'p50': 1909.1958431527019, 'p95': 2019.107734784484, 'max': 2019.2912109196186}, 'ttfb_ms': {'avg': 648.8797442871146, 'p50': 623.0050092563033, 'p95': 675.6928712129593, 'max': 675.8464118465781}, 'chunks_per_req': {'avg': 5.03125, 'p50': 5.0, 'p95': 5.0, 'max': 6.0}, 'inter_chunk_ms': {'avg': 275.22870759631314, 'p50': 207.84788858145475, 'p95': 740.5611611902714, 'max': 879.0481938049197}, 'tail_emit_speed_x': {'avg': 3.197858069350033, 'p50': 2.923360904858434, 'p95': 4.4770304382987005, 'max': 4.708444478225385}, 'gpu': {'samples': 4, 'memory_mb': {'min': 23679.0, 'avg': 25281.5, 'max': 26515.0, 'total': 81920.0}, 'gpu_util_pct': {'min': 6.0, 'avg': 65.75, 'max': 97.0}}, 'timing': {'timeline_to_first_batch_ms': {'avg': 35.23362666601315, 'p50': 35.25775112211704, 'p95': 39.834958501160145, 'max': 40.65745323896408}, 'timeline_to_first_chunk_emitted_ms': {'avg': 640.9386711602565, 'p50': 621.1660718545318, 'p95': 665.8211397007108, 'max': 666.3527227938175}, 'timeline_total_ms': {'avg': 1750.4810283717234, 'p50': 1781.4617110416293, 'p95': 2008.0293603241444, 'max': 2011.0769141465425}, 'llm_time_in_queue_ms': {'avg': 0.08527189493179321, 'p50': 0.08490215986967087, 'p95': 0.1421002671122551, 'max': 0.1627076417207718}, 'llm_scheduler_ms': {'avg': 26.065500918775797, 'p50': 25.585444644093513, 'p95': 25.585444644093513, 'max': 33.26634503901005}, 'llm_first_token_ms': {'avg': 34.08139944076538, 'p50': 34.08551216125488, 'p95': 38.91777992248535, 'max': 39.77704048156738}, 'llm_request_lifecycle_ms': {'avg': 1382.0002908469178, 'p50': 1379.5620054006577, 'p95': 1493.0487405508757, 'max': 1603.7994865328074}, 'llm_generation_wall_ms': {'avg': 1726.749652356375, 'p50': 1748.141685500741, 'p95': 2007.8193033114076, 'max': 2010.8438143506646}, 'llm_time_per_token_ms': {'avg': 7.72462807878853, 'p50': 7.814938672433753, 'p95': 8.604908601856538, 'max': 8.718051332170548}, 'llm_batch_wall_ms_total': {'avg': 1673.9102489082143, 'p50': 1703.9339151233435, 'p95': 1894.464598968625, 'max': 1977.4081986397505}, 'llm_decode_wall_ms_total': {'avg': 1638.7110949144699, 'p50': 1667.301645502448, 'p95': 1864.2176864668727, 'max': 1940.3332509100437}, 'tokens_per_batch_p50': {'avg': 1.0, 'p50': 1.0, 'p95': 1.0, 'max': 1.0}, 'bicodec_decode_calls': {'avg': 4.03125, 'p50': 4.0, 'p95': 4.0, 'max': 5.0}, 'bicodec_decode_wall_ms_total': {'avg': 623.9436317118816, 'p50': 690.9038424491882, 'p95': 871.2713327258825, 'max': 871.9572713598609}, 'bicodec_decode_interval_ms_p50': {'avg': 336.5506164554972, 'p50': 342.6840277388692, 'p95': 394.3607332184911, 'max': 394.45706456899643}, 'batch_avg_size': {'avg': 12.155133928571429, 'p50': 12.25, 'p95': 12.25, 'max': 12.25}, 'batch_max_seen': {'avg': 16.0, 'p50': 16.0, 'p95': 16.0, 'max': 16.0}, 'latency_over_timeline_ms': {'avg': 83.1321751466021, 'p50': 40.88493622839451, 'p95': 249.1534696891904, 'max': 312.9643900319934}}, 'requests': [{'success': True, 'latency_ms': 1909.4989970326424, 'ttfb_ms': 621.9156505540013, 'chunks_sent': 5, 'total_bytes': 118444, 'audio_seconds': 3.7, 'speaker': 'lipakshi', 'error': None, 'inter_chunk_ms': [173.0956146493554, 342.5519047304988, 459.67068057507277, 0.004678964614868164], 'tail_emit_speed_x': 2.8736003848752616, 'final_metrics': {'ttfb_ms': 621.1655912920833, 'chunks_sent': 5, 'total_bytes': 118444, 'audio_duration_seconds': 3.7, 'text_chunked': False, 'prompt_build_ms': 0.055145472288131714, 'sampling_params_ms': 0.060393474996089935, 'llm_generation_wall_ms': 1560.2657031267881, 'llm_batch_count': 140, 'llm_token_total': 220, 'semantic_token_total': 185, 'global_token_total': 32, 'llm_parse_ms': 0.4243701696395874, 'bicodec_decode_calls': 4, 'llm_decode_calls': 4, 'stream_output_kind': 'delta', 'stream_decode_interval_tokens': 48, 'stream_window_size_tokens': 128, 'stream_windowed_decode': True, 'timeline_markers': [{'stage': 'request_start', 't_ms': 0.0, 'dt_ms': 0.0}, {'stage': 'prompt_ready', 't_ms': 0.0576898455619812, 'dt_ms': 0.0576898455619812}, {'stage': 'sampling_ready', 't_ms': 0.11959578841924667, 'dt_ms': 0.06190594285726547}, {'stage': 'llm_first_batch', 't_ms': 37.426053546369076, 'dt_ms': 37.30645775794983}, {'stage': 'first_chunk_ready', 't_ms': 237.25101724267006, 'dt_ms': 199.82496369630098}, {'stage': 'first_chunk_decoded', 't_ms': 621.16255518049, 'dt_ms': 383.91153793781996}, {'stage': 'first_chunk_emitted', 't_ms': 621.1660718545318, 'dt_ms': 0.003516674041748047}, {'stage': 'llm_done', 't_ms': 1560.4411959648132, 'dt_ms': 939.2751241102815}, {'stage': 'stream_done', 't_ms': 1596.5193687006831, 'dt_ms': 36.078172735869884}, {'stage': 'request_done', 't_ms': 1596.5219531208277, 'dt_ms': 0.002584420144557953}], 'timeline_total_ms': 1596.534607000649, 'enabled': 1.0, 'batch_total_batches': 8.0, 'batch_total_requests': 98.0, 'batch_avg_size': 12.25, 'batch_max_seen': 16.0, 'llm_batch_wall_ms_total': 1560.2231016382575, 'llm_batch_wall_ms_min': 2.436739392578602, 'llm_batch_wall_ms_max': 200.90503711253405, 'llm_batch_wall_ms_p50': 4.606903530657291, 'llm_decode_wall_ms_total': 1522.97396492213, 'llm_decode_wall_ms_min': 2.436739392578602, 'llm_decode_wall_ms_max': 200.90503711253405, 'llm_decode_wall_ms_p50': 4.606903530657291, 'tokens_per_batch_total': 220.0, 'tokens_per_batch_min': 1.0, 'tokens_per_batch_max': 37.0, 'tokens_per_batch_p50': 1.0, 'llm_parse_step_ms_total': 0.4243701696395874, 'llm_parse_step_ms_min': 0.0011920928955078125, 'llm_parse_step_ms_max': 0.06586406379938126, 'llm_parse_step_ms_p50': 0.001884065568447113, 'bicodec_decode_wall_ms_total': 446.18992041796446, 'bicodec_decode_wall_ms_min': 35.95940861850977, 'bicodec_decode_wall_ms_max': 200.8170010522008, 'bicodec_decode_wall_ms_p50': 94.7122210636735, 'bicodec_decode_tokens_total': 457.0, 'bicodec_decode_tokens_min': 48.0, 'bicodec_decode_tokens_max': 185.0, 'bicodec_decode_tokens_p50': 96.0, 'bicodec_decode_interval_ms_total': 975.2618437632918, 'bicodec_decode_interval_ms_min': 172.94391803443432, 'bicodec_decode_interval_ms_max': 459.76755395531654, 'bicodec_decode_interval_ms_p50': 342.550371773541, 'bicodec_decode_wall_ms': 446.18992041796446, 'bicodec_decode_gpu_ms': 0.0, 'bicodec_decode_cpu_ms': 446.18992041796446, 'llm_time_per_token_ms': 7.092116832394492, 'llm_time_per_batch_wall_ms': 11.144450725987554, 'timeline_request_start_ms': 0.0, 'timeline_request_start_delta_ms': 0.0, 'timeline_prompt_ready_ms': 0.0576898455619812, 'timeline_prompt_ready_delta_ms': 0.0576898455619812, 'timeline_sampling_ready_ms': 0.11959578841924667, 'timeline_sampling_ready_delta_ms': 0.06190594285726547, 'timeline_llm_first_batch_ms': 37.426053546369076, 'timeline_llm_first_batch_delta_ms': 37.30645775794983, 'timeline_first_chunk_ready_ms': 237.25101724267006, 'timeline_first_chunk_ready_delta_ms': 199.82496369630098, 'timeline_first_chunk_decoded_ms': 621.16255518049, 'timeline_first_chunk_decoded_delta_ms': 383.91153793781996, 'timeline_first_chunk_emitted_ms': 621.1660718545318, 'timeline_first_chunk_emitted_delta_ms': 0.003516674041748047, 'timeline_llm_done_ms': 1560.4411959648132, 'timeline_llm_done_delta_ms': 939.2751241102815, 'timeline_stream_done_ms': 1596.5193687006831, 'timeline_stream_done_delta_ms': 36.078172735869884, 'timeline_request_done_ms': 1596.5219531208277, 'timeline_request_done_delta_ms': 0.002584420144557953, 'timeline_to_first_batch_ms': 37.426053546369076, 'timeline_to_first_chunk_emitted_ms': 621.1660718545318, 'timeline_first_batch_to_first_chunk_emitted_ms': 583.7400183081627, 'timeline_first_chunk_to_request_done_ms': 975.3558812662959, 'llm_first_token_ms': 36.74602508544922, 'llm_queued_to_scheduled_ms': 0.05647726356983185, 'llm_time_in_queue_ms': 0.05647726356983185, 'llm_scheduled_to_first_token_ms': 33.26634503901005, 'llm_scheduler_ms': 33.26634503901005, 'llm_first_to_last_token_ms': 1345.879846252501, 'llm_model_execute_ms': 1345.879846252501, 'llm_queued_to_last_token_ms': 1379.202668555081, 'llm_request_lifecycle_ms': 1379.202668555081}}, {'success': True, 'latency_ms': 1552.0446533337235, 'ttfb_ms': 622.0637299120426, 'chunks_sent': 5, 'total_bytes': 112684, 'audio_seconds': 3.52, 'speaker': 'vardan', 'error': None, 'inter_chunk_ms': [173.0642467737198, 342.5666429102421, 264.6071007475257, 0.0037569552659988403], 'tail_emit_speed_x': 3.7850238766714224, 'final_metrics': {'ttfb_ms': 619.6328410878778, 'chunks_sent': 5, 'total_bytes': 112684, 'audio_duration_seconds': 3.52, 'text_chunked': False, 'prompt_build_ms': 0.009778887033462524, 'sampling_params_ms': 0.00918656587600708, 'llm_generation_wall_ms': 1365.0940107181668, 'llm_batch_count': 139, 'llm_token_total': 211, 'semantic_token_total': 176, 'global_token_total': 32, 'llm_parse_ms': 0.30895695090293884, 'bicodec_decode_calls': 4, 'llm_decode_calls': 4, 'stream_output_kind': 'delta', 'stream_decode_interval_tokens': 48, 'stream_window_size_tokens': 128, 'stream_windowed_decode': True, 'timeline_markers': [{'stage': 'request_start', 't_ms': 0.0, 'dt_ms': 0.0}, {'stage': 'prompt_ready', 't_ms': 0.011301599442958832, 'dt_ms': 0.011301599442958832}, {'stage': 'sampling_ready', 't_ms': 0.02174079418182373, 'dt_ms': 0.010439194738864899}, {'stage': 'llm_first_batch', 't_ms': 35.75554210692644, 'dt_ms': 35.73380131274462}, {'stage': 'first_chunk_ready', 't_ms': 235.54631136357784, 'dt_ms': 199.7907692566514}, {'stage': 'first_chunk_decoded', 't_ms': 619.6292340755463, 'dt_ms': 384.0829227119684}, {'stage': 'first_chunk_emitted', 't_ms': 619.6330115199089, 'dt_ms': 0.003777444362640381}, {'stage': 'llm_done', 't_ms': 1365.1282154023647, 'dt_ms': 745.4952038824558}, {'stage': 'stream_done', 't_ms': 1399.8877936974168, 'dt_ms': 34.75957829505205}, {'stage': 'request_done', 't_ms': 1399.8905289918184, 'dt_ms': 0.0027352944016456604}], 'timeline_total_ms': 1399.9040136113763, 'enabled': 1.0, 'batch_total_batches': 8.0, 'batch_total_requests': 98.0, 'batch_avg_size': 12.25, 'batch_max_seen': 16.0, 'llm_batch_wall_ms_total': 1365.0588635355234, 'llm_batch_wall_ms_min': 2.326550893485546, 'llm_batch_wall_ms_max': 201.02544408291578, 'llm_batch_wall_ms_p50': 4.6065133064985275, 'llm_decode_wall_ms_total': 1329.3379861861467, 'llm_decode_wall_ms_min': 2.326550893485546, 'llm_decode_wall_ms_max': 201.02544408291578, 'llm_decode_wall_ms_p50': 4.603054374456406, 'tokens_per_batch_total': 211.0, 'tokens_per_batch_min': 1.0, 'tokens_per_batch_max': 37.0, 'tokens_per_batch_p50': 1.0, 'llm_parse_step_ms_total': 0.30895695090293884, 'llm_parse_step_ms_min': 0.0006416812539100647, 'llm_parse_step_ms_max': 0.056996941566467285, 'llm_parse_step_ms_p50': 0.001473352313041687, 'bicodec_decode_wall_ms_total': 445.52283827215433, 'bicodec_decode_wall_ms_min': 34.62923318147659, 'bicodec_decode_wall_ms_max': 200.97628142684698, 'bicodec_decode_wall_ms_p50': 94.95668206363916, 'bicodec_decode_tokens_total': 448.0, 'bicodec_decode_tokens_min': 48.0, 'bicodec_decode_tokens_max': 176.0, 'bicodec_decode_tokens_p50': 96.0, 'bicodec_decode_interval_ms_total': 780.1422895863652, 'bicodec_decode_interval_ms_min': 173.03101159632206, 'bicodec_decode_interval_ms_max': 342.5664948299527, 'bicodec_decode_interval_ms_p50': 264.54478316009045, 'bicodec_decode_wall_ms': 445.52283827215433, 'bicodec_decode_gpu_ms': 0.0, 'bicodec_decode_cpu_ms': 445.52283827215433, 'llm_time_per_token_ms': 6.4696398612235395, 'llm_time_per_batch_wall_ms': 9.820567363564917, 'timeline_request_start_ms': 0.0, 'timeline_request_start_delta_ms': 0.0, 'timeline_prompt_ready_ms': 0.011301599442958832, 'timeline_prompt_ready_delta_ms': 0.011301599442958832, 'timeline_sampling_ready_ms': 0.02174079418182373, 'timeline_sampling_ready_delta_ms': 0.010439194738864899, 'timeline_llm_first_batch_ms': 35.75554210692644, 'timeline_llm_first_batch_delta_ms': 35.73380131274462, 'timeline_first_chunk_ready_ms': 235.54631136357784, 'timeline_first_chunk_ready_delta_ms': 199.7907692566514, 'timeline_first_chunk_decoded_ms': 619.6292340755463, 'timeline_first_chunk_decoded_delta_ms': 384.0829227119684, 'timeline_first_chunk_emitted_ms': 619.6330115199089, 'timeline_first_chunk_emitted_delta_ms': 0.003777444362640381, 'timeline_llm_done_ms': 1365.1282154023647, 'timeline_llm_done_delta_ms': 745.4952038824558, 'timeline_stream_done_ms': 1399.8877936974168, 'timeline_stream_done_delta_ms': 34.75957829505205, 'timeline_request_done_ms': 1399.8905289918184, 'timeline_request_done_delta_ms': 0.0027352944016456604, 'timeline_to_first_batch_ms': 35.75554210692644, 'timeline_to_first_chunk_emitted_ms': 619.6330115199089, 'timeline_first_batch_to_first_chunk_emitted_ms': 583.8774694129825, 'timeline_first_chunk_to_request_done_ms': 780.2575174719095, 'llm_first_token_ms': 35.30740737915039, 'llm_queued_to_scheduled_ms': 0.022563152015209198, 'llm_time_in_queue_ms': 0.022563152015209198, 'llm_scheduled_to_first_token_ms': 33.26634503901005, 'llm_scheduler_ms': 33.26634503901005, 'llm_first_to_last_token_ms': 1292.7590869367123, 'llm_model_execute_ms': 1292.7590869367123, 'llm_queued_to_last_token_ms': 1326.0479951277375, 'llm_request_lifecycle_ms': 1326.0479951277375}}, {'success': True, 'latency_ms': 1276.3068368658423, 'ttfb_ms': 622.1629865467548, 'chunks_sent': 5, 'total_bytes': 98604, 'audio_seconds': 3.08, 'speaker': 'reet', 'error': None, 'inter_chunk_ms': [173.06264396756887, 342.5629260018468, 137.37141992896795, 0.005641952157020569], 'tail_emit_speed_x': 4.708444478225385, 'final_metrics': {'ttfb_ms': 619.2943202331662, 'chunks_sent': 5, 'total_bytes': 98604, 'audio_duration_seconds': 3.08, 'text_chunked': False, 'prompt_build_ms': 0.008384697139263153, 'sampling_params_ms': 0.00873766839504242, 'llm_generation_wall_ms': 1143.1785123422742, 'llm_batch_count': 139, 'llm_token_total': 189, 'semantic_token_total': 154, 'global_token_total': 32, 'llm_parse_ms': 0.29246043413877487, 'bicodec_decode_calls': 4, 'llm_decode_calls': 4, 'stream_output_kind': 'delta', 'stream_decode_interval_tokens': 48, 'stream_window_size_tokens': 128, 'stream_windowed_decode': True, 'timeline_markers': [{'stage': 'request_start', 't_ms': 0.0, 'dt_ms': 0.0}, {'stage': 'prompt_ready', 't_ms': 0.009867362678050995, 'dt_ms': 0.009867362678050995}, {'stage': 'sampling_ready', 't_ms': 0.01975707709789276, 'dt_ms': 0.009889714419841766}, {'stage': 'llm_first_batch', 't_ms': 40.65745323896408, 'dt_ms': 40.63769616186619}, {'stage': 'first_chunk_ready', 't_ms': 240.08489679545164, 'dt_ms': 199.42744355648756}, {'stage': 'first_chunk_decoded', 't_ms': 619.2925963550806, 'dt_ms': 379.20769955962896}, {'stage': 'first_chunk_emitted', 't_ms': 619.294460862875, 'dt_ms': 0.001864507794380188}, {'stage': 'llm_done', 't_ms': 1143.2099407538772, 'dt_ms': 523.9154798910022}, {'stage': 'stream_done', 't_ms': 1272.3083039745688, 'dt_ms': 129.09836322069168}, {'stage': 'request_done', 't_ms': 1272.3124120384455, 'dt_ms': 0.004108063876628876}], 'timeline_total_ms': 1272.3380597308278, 'enabled': 1.0, 'batch_total_batches': 7.0, 'batch_total_requests': 82.0, 'batch_avg_size': 11.714285714285714, 'batch_max_seen': 16.0, 'llm_batch_wall_ms_total': 1143.1549480184913, 'llm_batch_wall_ms_min': 2.2353166714310646, 'llm_batch_wall_ms_max': 196.58226985484362, 'llm_batch_wall_ms_p50': 4.600080661475658, 'llm_decode_wall_ms_total': 1102.5288319215178, 'llm_decode_wall_ms_min': 2.2353166714310646, 'llm_decode_wall_ms_max': 196.58226985484362, 'llm_decode_wall_ms_p50': 4.59987111389637, 'tokens_per_batch_total': 189.0, 'tokens_per_batch_min': 1.0, 'tokens_per_batch_max': 36.0, 'tokens_per_batch_p50': 1.0, 'llm_parse_step_ms_total': 0.29246043413877487, 'llm_parse_step_ms_min': 0.00067148357629776, 'llm_parse_step_ms_max': 0.06199721246957779, 'llm_parse_step_ms_p50': 0.0014435499906539917, 'bicodec_decode_wall_ms_total': 526.454652659595, 'bicodec_decode_wall_ms_min': 90.69284703582525, 'bicodec_decode_wall_ms_max': 196.5319663286209, 'bicodec_decode_wall_ms_p50': 110.27529835700989, 'bicodec_decode_tokens_total': 426.0, 'bicodec_decode_tokens_min': 48.0, 'bicodec_decode_tokens_max': 154.0, 'bicodec_decode_tokens_p50': 96.0, 'bicodec_decode_interval_ms_total': 652.886881493032, 'bicodec_decode_interval_ms_min': 137.29284331202507, 'bicodec_decode_interval_ms_max': 342.5753405317664, 'bicodec_decode_interval_ms_p50': 173.0186976492405, 'bicodec_decode_wall_ms': 526.454652659595, 'bicodec_decode_gpu_ms': 0.0, 'bicodec_decode_cpu_ms': 526.454652659595, 'llm_time_per_token_ms': 6.04856355736653, 'llm_time_per_batch_wall_ms': 8.224136316679793, 'timeline_request_start_ms': 0.0, 'timeline_request_start_delta_ms': 0.0, 'timeline_prompt_ready_ms': 0.009867362678050995, 'timeline_prompt_ready_delta_ms': 0.009867362678050995, 'timeline_sampling_ready_ms': 0.01975707709789276, 'timeline_sampling_ready_delta_ms': 0.009889714419841766, 'timeline_llm_first_batch_ms': 40.65745323896408, 'timeline_llm_first_batch_delta_ms': 40.63769616186619, 'timeline_first_chunk_ready_ms': 240.08489679545164, 'timeline_first_chunk_ready_delta_ms': 199.42744355648756, 'timeline_first_chunk_decoded_ms': 619.2925963550806, 'timeline_first_chunk_decoded_delta_ms': 379.20769955962896, 'timeline_first_chunk_emitted_ms': 619.294460862875, 'timeline_first_chunk_emitted_delta_ms': 0.001864507794380188, 'timeline_llm_done_ms': 1143.2099407538772, 'timeline_llm_done_delta_ms': 523.9154798910022, 'timeline_stream_done_ms': 1272.3083039745688, 'timeline_stream_done_delta_ms': 129.09836322069168, 'timeline_request_done_ms': 1272.3124120384455, 'timeline_request_done_delta_ms': 0.004108063876628876, 'timeline_to_first_batch_ms': 40.65745323896408, 'timeline_to_first_chunk_emitted_ms': 619.294460862875, 'timeline_first_batch_to_first_chunk_emitted_ms': 578.6370076239109, 'timeline_first_chunk_to_request_done_ms': 653.0179511755705, 'llm_first_token_ms': 39.77704048156738, 'llm_queued_to_scheduled_ms': 0.1627076417207718, 'llm_time_in_queue_ms': 0.1627076417207718, 'llm_scheduled_to_first_token_ms': 25.585444644093513, 'llm_scheduler_ms': 25.585444644093513, 'llm_first_to_last_token_ms': 1102.439682930708, 'llm_model_execute_ms': 1102.439682930708, 'llm_queued_to_last_token_ms': 1128.1878352165222, 'llm_request_lifecycle_ms': 1128.1878352165222}}, {'success': True, 'latency_ms': 1552.0478188991547, 'ttfb_ms': 622.2620354965329, 'chunks_sent': 5, 'total_bytes': 110764, 'audio_seconds': 3.46, 'speaker': 'Nandini', 'error': None, 'inter_chunk_ms': [173.0411807075143, 342.5731174647808, 300.8816549554467, 0.0035669654607772827], 'tail_emit_speed_x': 3.721287270426815, 'final_metrics': {'ttfb_ms': 618.9637761563063, 'chunks_sent': 5, 'total_bytes': 110764, 'audio_duration_seconds': 3.46, 'text_chunked': False, 'prompt_build_ms': 0.007724389433860779, 'sampling_params_ms': 0.008315779268741608, 'llm_generation_wall_ms': 1399.1967169567943, 'llm_batch_count': 141, 'llm_token_total': 208, 'semantic_token_total': 173, 'global_token_total': 32, 'llm_parse_ms': 0.301554799079895, 'bicodec_decode_calls': 4, 'llm_decode_calls': 4, 'stream_output_kind': 'delta', 'stream_decode_interval_tokens': 48, 'stream_window_size_tokens': 128, 'stream_windowed_decode': True, 'timeline_markers': [{'stage': 'request_start', 't_ms': 0.0, 'dt_ms': 0.0}, {'stage': 'prompt_ready', 't_ms': 0.009238719940185547, 'dt_ms': 0.009238719940185547}, {'stage': 'sampling_ready', 't_ms': 0.018705613911151886, 'dt_ms': 0.009466893970966339}, {'stage': 'llm_first_batch', 't_ms': 40.23769125342369, 'dt_ms': 40.21898563951254}, {'stage': 'first_chunk_ready', 't_ms': 239.66054432094097, 'dt_ms': 199.42285306751728}, {'stage': 'first_chunk_decoded', 't_ms': 618.9623745158315, 'dt_ms': 379.3018301948905}, {'stage': 'first_chunk_emitted', 't_ms': 618.9639065414667, 'dt_ms': 0.0015320256352424622}, {'stage': 'llm_done', 't_ms': 1399.2255916818976, 'dt_ms': 780.2616851404309}, {'stage': 'stream_done', 't_ms': 1435.4734849184752, 'dt_ms': 36.24789323657751}, {'stage': 'request_done', 't_ms': 1435.4757405817509, 'dt_ms': 0.002255663275718689}], 'timeline_total_ms': 1435.48774253577, 'enabled': 1.0, 'batch_total_batches': 8.0, 'batch_total_requests': 98.0, 'batch_avg_size': 12.25, 'batch_max_seen': 16.0, 'llm_batch_wall_ms_total': 1399.1403495892882, 'llm_batch_wall_ms_min': 2.1618297323584557, 'llm_batch_wall_ms_max': 196.65884505957365, 'llm_batch_wall_ms_p50': 4.60752472281456, 'llm_decode_wall_ms_total': 1358.9322660118341, 'llm_decode_wall_ms_min': 2.1618297323584557, 'llm_decode_wall_ms_max': 196.65884505957365, 'llm_decode_wall_ms_p50': 4.601503722369671, 'tokens_per_batch_total': 208.0, 'tokens_per_batch_min': 1.0, 'tokens_per_batch_max': 36.0, 'tokens_per_batch_p50': 1.0, 'llm_parse_step_ms_total': 0.301554799079895, 'llm_parse_step_ms_min': 0.0006007030606269836, 'llm_parse_step_ms_max': 0.046026892960071564, 'llm_parse_step_ms_p50': 0.0013820827007293701, 'bicodec_decode_wall_ms_total': 433.8765777647495, 'bicodec_decode_wall_ms_min': 36.14795487374067, 'bicodec_decode_wall_ms_max': 196.61484193056822, 'bicodec_decode_wall_ms_p50': 90.76848067343235, 'bicodec_decode_tokens_total': 445.0, 'bicodec_decode_tokens_min': 48.0, 'bicodec_decode_tokens_max': 173.0, 'bicodec_decode_tokens_p50': 96.0, 'bicodec_decode_interval_ms_total': 816.4247181266546, 'bicodec_decode_interval_ms_min': 173.01446106284857, 'bicodec_decode_interval_ms_max': 342.5706513226032, 'bicodec_decode_interval_ms_p50': 300.83960574120283, 'bicodec_decode_wall_ms': 433.8765777647495, 'bicodec_decode_gpu_ms': 0.0, 'bicodec_decode_cpu_ms': 433.8765777647495, 'llm_time_per_token_ms': 6.726907293061511, 'llm_time_per_batch_wall_ms': 9.92298120276091, 'timeline_request_start_ms': 0.0, 'timeline_request_start_delta_ms': 0.0, 'timeline_prompt_ready_ms': 0.009238719940185547, 'timeline_prompt_ready_delta_ms': 0.009238719940185547, 'timeline_sampling_ready_ms': 0.018705613911151886, 'timeline_sampling_ready_delta_ms': 0.009466893970966339, 'timeline_llm_first_batch_ms': 40.23769125342369, 'timeline_llm_first_batch_delta_ms': 40.21898563951254, 'timeline_first_chunk_ready_ms': 239.66054432094097, 'timeline_first_chunk_ready_delta_ms': 199.42285306751728, 'timeline_first_chunk_decoded_ms': 618.9623745158315, 'timeline_first_chunk_decoded_delta_ms': 379.3018301948905, 'timeline_first_chunk_emitted_ms': 618.9639065414667, 'timeline_first_chunk_emitted_delta_ms': 0.0015320256352424622, 'timeline_llm_done_ms': 1399.2255916818976, 'timeline_llm_done_delta_ms': 780.2616851404309, 'timeline_stream_done_ms': 1435.4734849184752, 'timeline_stream_done_delta_ms': 36.24789323657751, 'timeline_request_done_ms': 1435.4757405817509, 'timeline_request_done_delta_ms': 0.002255663275718689, 'timeline_to_first_batch_ms': 40.23769125342369, 'timeline_to_first_chunk_emitted_ms': 618.9639065414667, 'timeline_first_batch_to_first_chunk_emitted_ms': 578.726215288043, 'timeline_first_chunk_to_request_done_ms': 816.5118340402842, 'llm_first_token_ms': 39.34597969055176, 'llm_queued_to_scheduled_ms': 0.15581492334604263, 'llm_time_in_queue_ms': 0.15581492334604263, 'llm_scheduled_to_first_token_ms': 25.585444644093513, 'llm_scheduler_ms': 25.585444644093513, 'llm_first_to_last_token_ms': 1275.2062054350972, 'llm_model_execute_ms': 1275.2062054350972, 'llm_queued_to_last_token_ms': 1300.9474650025368, 'llm_request_lifecycle_ms': 1300.9474650025368}}, {'success': True, 'latency_ms': 1360.3219641372561, 'ttfb_ms': 622.343678958714, 'chunks_sent': 5, 'total_bytes': 106284, 'audio_seconds': 3.32, 'speaker': 'krishna', 'error': None, 'inter_chunk_ms': [173.03833737969398, 342.6014482975006, 183.974071405828, 0.0043781474232673645], 'tail_emit_speed_x': 4.498777357922908, 'final_metrics': {'ttfb_ms': 618.6349634081125, 'chunks_sent': 5, 'total_bytes': 106284, 'audio_duration_seconds': 3.32, 'text_chunked': False, 'prompt_build_ms': 0.008004717528820038, 'sampling_params_ms': 0.008…235003 tokens truncated…lm_done', 't_ms': 3917.768252082169, 'dt_ms': 2816.554536111653}, {'stage': 'stream_done', 't_ms': 3917.7722800523043, 'dt_ms': 0.004027970135211945}, {'stage': 'request_done', 't_ms': 3917.773060500622, 'dt_ms': 0.000780448317527771}], 'timeline_total_ms': 3917.7845930680633, 'enabled': 1.0, 'batch_total_batches': 72.0, 'batch_total_requests': 650.0, 'batch_avg_size': 9.027777777777779, 'batch_max_seen': 16.0, 'llm_batch_wall_ms_total': 3558.7747348472476, 'llm_batch_wall_ms_min': 3.3158529549837112, 'llm_batch_wall_ms_max': 1773.5230354592204, 'llm_batch_wall_ms_p50': 7.032112218439579, 'llm_decode_wall_ms_total': 3505.225625820458, 'llm_decode_wall_ms_min': 3.3158529549837112, 'llm_decode_wall_ms_max': 1773.5230354592204, 'llm_decode_wall_ms_p50': 7.032112218439579, 'tokens_per_batch_total': 247.0, 'tokens_per_batch_min': 1.0, 'tokens_per_batch_max': 63.0, 'tokens_per_batch_p50': 1.0, 'llm_parse_step_ms_total': 0.3002900630235672, 'llm_parse_step_ms_min': 0.000500120222568512, 'llm_parse_step_ms_max': 0.09292550384998322, 'llm_parse_step_ms_p50': 0.0013932585716247559, 'bicodec_decode_wall_ms_total': 3144.061424769461, 'bicodec_decode_wall_ms_min': 358.7777577340603, 'bicodec_decode_wall_ms_max': 1773.424107581377, 'bicodec_decode_wall_ms_p50': 465.5062071979046, 'bicodec_decode_tokens_total': 400.0, 'bicodec_decode_tokens_min': 48.0, 'bicodec_decode_tokens_max': 128.0, 'bicodec_decode_tokens_p50': 96.0, 'bicodec_decode_interval_ms_total': 2816.5075685828924, 'bicodec_decode_interval_ms_min': 358.9780982583761, 'bicodec_decode_interval_ms_max': 1903.3496342599392, 'bicodec_decode_interval_ms_p50': 554.1798360645771, 'bicodec_decode_wall_ms': 3144.061424769461, 'bicodec_decode_gpu_ms': 0.0, 'bicodec_decode_cpu_ms': 3144.061424769461, 'llm_time_per_token_ms': 15.861303202415767, 'llm_time_per_batch_wall_ms': 38.682334074426606, 'timeline_request_start_ms': 0.0, 'timeline_request_start_delta_ms': 0.0, 'timeline_prompt_ready_ms': 0.00813417136669159, 'timeline_prompt_ready_delta_ms': 0.00813417136669159, 'timeline_sampling_ready_ms': 0.01658126711845398, 'timeline_sampling_ready_delta_ms': 0.00844709575176239, 'timeline_llm_first_batch_ms': 53.5767013207078, 'timeline_llm_first_batch_delta_ms': 53.560120053589344, 'timeline_first_chunk_ready_ms': 368.7482066452503, 'timeline_first_chunk_ready_delta_ms': 315.1715053245425, 'timeline_first_chunk_decoded_ms': 1101.2118412181735, 'timeline_first_chunk_decoded_delta_ms': 732.4636345729232, 'timeline_first_chunk_emitted_ms': 1101.2137159705162, 'timeline_first_chunk_emitted_delta_ms': 0.0018747523427009583, 'timeline_llm_done_ms': 3917.768252082169, 'timeline_llm_done_delta_ms': 2816.554536111653, 'timeline_stream_done_ms': 3917.7722800523043, 'timeline_stream_done_delta_ms': 0.004027970135211945, 'timeline_request_done_ms': 3917.773060500622, 'timeline_request_done_delta_ms': 0.000780448317527771, 'timeline_to_first_batch_ms': 53.5767013207078, 'timeline_to_first_chunk_emitted_ms': 1101.2137159705162, 'timeline_first_batch_to_first_chunk_emitted_ms': 1047.6370146498084, 'timeline_first_chunk_to_request_done_ms': 2816.5593445301056, 'llm_first_token_ms': 47.21426963806152, 'llm_queued_to_scheduled_ms': 0.022361986339092255, 'llm_time_in_queue_ms': 0.022361986339092255, 'llm_scheduled_to_first_token_ms': 30.961014330387115, 'llm_scheduler_ms': 30.961014330387115, 'llm_first_to_last_token_ms': 2320.498493500054, 'llm_model_execute_ms': 2320.498493500054, 'llm_queued_to_last_token_ms': 2351.48186981678, 'llm_request_lifecycle_ms': 2351.48186981678}}, {'success': True, 'latency_ms': 3953.135429881513, 'ttfb_ms': 1136.1040314659476, 'chunks_sent': 5, 'total_bytes': 119724, 'audio_seconds': 3.74, 'speaker': 'reet', 'error': None, 'inter_chunk_ms': [554.2368832975626, 1903.4394454210997, 358.927289955318, 0.011602416634559631], 'tail_emit_speed_x': 1.3276387342020954, 'final_metrics': {'ttfb_ms': 1100.9777588769794, 'chunks_sent': 5, 'total_bytes': 119724, 'audio_duration_seconds': 3.74, 'text_chunked': False, 'prompt_build_ms': 0.006992369890213013, 'sampling_params_ms': 0.007315538823604584, 'llm_generation_wall_ms': 3917.577400803566, 'llm_batch_count': 92, 'llm_token_total': 264, 'semantic_token_total': 229, 'global_token_total': 32, 'llm_parse_ms': 0.30393805354833603, 'bicodec_decode_calls': 4, 'llm_decode_calls': 4, 'stream_output_kind': 'delta', 'stream_decode_interval_tokens': 48, 'stream_window_size_tokens': 128, 'stream_windowed_decode': True, 'timeline_markers': [{'stage': 'request_start', 't_ms': 0.0, 'dt_ms': 0.0}, {'stage': 'prompt_ready', 't_ms': 0.008306466042995453, 'dt_ms': 0.008306466042995453}, {'stage': 'sampling_ready', 't_ms': 0.01666322350502014, 'dt_ms': 0.008356757462024689}, {'stage': 'llm_first_batch', 't_ms': 53.26605401933193, 'dt_ms': 53.24939079582691}, {'stage': 'first_chunk_ready', 't_ms': 368.43678914010525, 'dt_ms': 315.1707351207733}, {'stage': 'first_chunk_decoded', 't_ms': 1100.9758152067661, 'dt_ms': 732.5390260666609}, {'stage': 'first_chunk_emitted', 't_ms': 1100.9778892621398, 'dt_ms': 0.0020740553736686707}, {'stage': 'llm_done', 't_ms': 3917.603430338204, 'dt_ms': 2816.625541076064}, {'stage': 'stream_done', 't_ms': 3917.6078783348203, 'dt_ms': 0.004447996616363525}, {'stage': 'request_done', 't_ms': 3917.608479037881, 'dt_ms': 0.0006007030606269836}], 'timeline_total_ms': 3917.615703307092, 'enabled': 1.0, 'batch_total_batches': 72.0, 'batch_total_requests': 650.0, 'batch_avg_size': 9.027777777777779, 'batch_max_seen': 16.0, 'llm_batch_wall_ms_total': 3558.6513923481107, 'llm_batch_wall_ms_min': 3.3198799937963486, 'llm_batch_wall_ms_max': 1773.7045781686902, 'llm_batch_wall_ms_p50': 7.026440463960171, 'llm_decode_wall_ms_total': 3505.4131615906954, 'llm_decode_wall_ms_min': 3.3198799937963486, 'llm_decode_wall_ms_max': 1773.7045781686902, 'llm_decode_wall_ms_p50': 7.026440463960171, 'tokens_per_batch_total': 264.0, 'tokens_per_batch_min': 1.0, 'tokens_per_batch_max': 80.0, 'tokens_per_batch_p50': 1.0, 'llm_parse_step_ms_total': 0.30393805354833603, 'llm_parse_step_ms_min': 0.0005112960934638977, 'llm_parse_step_ms_max': 0.1076841726899147, 'llm_parse_step_ms_p50': 0.0012926757335662842, 'bicodec_decode_wall_ms_total': 3144.404023885727, 'bicodec_decode_wall_ms_min': 358.7286351248622, 'bicodec_decode_wall_ms_max': 1773.6231153830886, 'bicodec_decode_wall_ms_p50': 465.58575704693794, 'bicodec_decode_tokens_total': 400.0, 'bicodec_decode_tokens_min': 48.0, 'bicodec_decode_tokens_max': 128.0, 'bicodec_decode_tokens_p50': 96.0, 'bicodec_decode_interval_ms_total': 2816.5835021063685, 'bicodec_decode_interval_ms_min': 358.92531648278236, 'bicodec_decode_interval_ms_max': 1903.4452168270946, 'bicodec_decode_interval_ms_p50': 554.2129687964916, 'bicodec_decode_wall_ms': 3144.404023885727, 'bicodec_decode_gpu_ms': 0.0, 'bicodec_decode_cpu_ms': 3144.404023885727, 'llm_time_per_token_ms': 14.839308336377144, 'llm_time_per_batch_wall_ms': 38.68099339508816, 'timeline_request_start_ms': 0.0, 'timeline_request_start_delta_ms': 0.0, 'timeline_prompt_ready_ms': 0.008306466042995453, 'timeline_prompt_ready_delta_ms': 0.008306466042995453, 'timeline_sampling_ready_ms': 0.01666322350502014, 'timeline_sampling_ready_delta_ms': 0.008356757462024689, 'timeline_llm_first_batch_ms': 53.26605401933193, 'timeline_llm_first_batch_delta_ms': 53.24939079582691, 'timeline_first_chunk_ready_ms': 368.43678914010525, 'timeline_first_chunk_ready_delta_ms': 315.1707351207733, 'timeline_first_chunk_decoded_ms': 1100.9758152067661, 'timeline_first_chunk_decoded_delta_ms': 732.5390260666609, 'timeline_first_chunk_emitted_ms': 1100.9778892621398, 'timeline_first_chunk_emitted_delta_ms': 0.0020740553736686707, 'timeline_llm_done_ms': 3917.603430338204, 'timeline_llm_done_delta_ms': 2816.625541076064, 'timeline_stream_done_ms': 3917.6078783348203, 'timeline_stream_done_delta_ms': 0.004447996616363525, 'timeline_request_done_ms': 3917.608479037881, 'timeline_request_done_delta_ms': 0.0006007030606269836, 'timeline_to_first_batch_ms': 53.26605401933193, 'timeline_to_first_chunk_emitted_ms': 1100.9778892621398, 'timeline_first_batch_to_first_chunk_emitted_ms': 1047.7118352428079, 'timeline_first_chunk_to_request_done_ms': 2816.630589775741, 'llm_first_token_ms': 46.8747615814209, 'llm_queued_to_scheduled_ms': 0.018154270946979523, 'llm_time_in_queue_ms': 0.018154270946979523, 'llm_scheduled_to_first_token_ms': 30.961014330387115, 'llm_scheduler_ms': 30.961014330387115, 'llm_first_to_last_token_ms': 2448.8340532407165, 'llm_model_execute_ms': 2448.8340532407165, 'llm_queued_to_last_token_ms': 2479.8132218420506, 'llm_request_lifecycle_ms': 2479.8132218420506}}, {'success': True, 'latency_ms': 3800.1137152314186, 'ttfb_ms': 1136.1894831061363, 'chunks_sent': 5, 'total_bytes': 112044, 'audio_seconds': 3.5, 'speaker': 'Nandini', 'error': None, 'inter_chunk_ms': [554.2646674439311, 1903.5240234807134, 12.242087163031101, 0.00303611159324646], 'tail_emit_speed_x': 1.3138511815734697, 'final_metrics': {'ttfb_ms': 1100.721805356443, 'chunks_sent': 5, 'total_bytes': 112044, 'audio_duration_seconds': 3.5, 'text_chunked': False, 'prompt_build_ms': 0.006882473826408386, 'sampling_params_ms': 0.007314607501029968, 'llm_generation_wall_ms': 3558.5735458880663, 'llm_batch_count': 92, 'llm_token_total': 210, 'semantic_token_total': 175, 'global_token_total': 32, 'llm_parse_ms': 0.23727770894765854, 'bicodec_decode_calls': 4, 'llm_decode_calls': 4, 'stream_output_kind': 'delta', 'stream_decode_interval_tokens': 48, 'stream_window_size_tokens': 128, 'stream_windowed_decode': True, 'timeline_markers': [{'stage': 'request_start', 't_ms': 0.0, 'dt_ms': 0.0}, {'stage': 'prompt_ready', 't_ms': 0.008205883204936981, 'dt_ms': 0.008205883204936981}, {'stage': 'sampling_ready', 't_ms': 0.016582198441028595, 'dt_ms': 0.008376315236091614}, {'stage': 'llm_first_batch', 't_ms': 52.93853487819433, 'dt_ms': 52.9219526797533}, {'stage': 'first_chunk_ready', 't_ms': 368.1029062718153, 'dt_ms': 315.16437139362097}, {'stage': 'first_chunk_decoded', 't_ms': 1100.720222108066, 'dt_ms': 732.6173158362508}, {'stage': 'first_chunk_emitted', 't_ms': 1100.7219245657325, 'dt_ms': 0.0017024576663970947}, {'stage': 'llm_done', 't_ms': 3558.602520264685, 'dt_ms': 2457.8805956989527}, {'stage': 'stream_done', 't_ms': 3570.770296268165, 'dt_ms': 12.167776003479958}, {'stage': 'request_done', 't_ms': 3570.7718394696712, 'dt_ms': 0.001543201506137848}], 'timeline_total_ms': 3570.7817366346717, 'enabled': 1.0, 'batch_total_batches': 69.0, 'batch_total_requests': 623.0, 'batch_avg_size': 9.028985507246377, 'batch_max_seen': 16.0, 'llm_batch_wall_ms_total': 3558.503082022071, 'llm_batch_wall_ms_min': 3.3239275217056274, 'llm_batch_wall_ms_max': 1773.8804416731, 'llm_batch_wall_ms_p50': 7.026472128927708, 'llm_decode_wall_ms_total': 3505.595245398581, 'llm_decode_wall_ms_min': 3.3239275217056274, 'llm_decode_wall_ms_max': 1773.8804416731, 'llm_decode_wall_ms_p50': 7.026472128927708, 'tokens_per_batch_total': 210.0, 'tokens_per_batch_min': 1.0, 'tokens_per_batch_max': 47.0, 'tokens_per_batch_p50': 1.0, 'llm_parse_step_ms_total': 0.23727770894765854, 'llm_parse_step_ms_min': 0.0005317851901054382, 'llm_parse_step_ms_max': 0.04416331648826599, 'llm_parse_step_ms_p50': 0.0013820827007293701, 'bicodec_decode_wall_ms_total': 2798.111852258444, 'bicodec_decode_wall_ms_min': 12.08709366619587, 'bicodec_decode_wall_ms_max': 1773.8007502630353, 'bicodec_decode_wall_ms_p50': 465.6595569103956, 'bicodec_decode_tokens_total': 447.0, 'bicodec_decode_tokens_min': 48.0, 'bicodec_decode_tokens_max': 175.0, 'bicodec_decode_tokens_p50': 96.0, 'bicodec_decode_interval_ms_total': 2469.9881272390485, 'bicodec_decode_interval_ms_min': 12.21572607755661, 'bicodec_decode_interval_ms_max': 1903.5352561622858, 'bicodec_decode_interval_ms_p50': 554.2371449992061, 'bicodec_decode_wall_ms': 2798.111852258444, 'bicodec_decode_gpu_ms': 0.0, 'bicodec_decode_cpu_ms': 2798.111852258444, 'llm_time_per_token_ms': 16.945588313752697, 'llm_time_per_batch_wall_ms': 38.679381326326855, 'timeline_request_start_ms': 0.0, 'timeline_request_start_delta_ms': 0.0, 'timeline_prompt_ready_ms': 0.008205883204936981, 'timeline_prompt_ready_delta_ms': 0.008205883204936981, 'timeline_sampling_ready_ms': 0.016582198441028595, 'timeline_sampling_ready_delta_ms': 0.008376315236091614, 'timeline_llm_first_batch_ms': 52.93853487819433, 'timeline_llm_first_batch_delta_ms': 52.9219526797533, 'timeline_first_chunk_ready_ms': 368.1029062718153, 'timeline_first_chunk_ready_delta_ms': 315.16437139362097, 'timeline_first_chunk_decoded_ms': 1100.720222108066, 'timeline_first_chunk_decoded_delta_ms': 732.6173158362508, 'timeline_first_chunk_emitted_ms': 1100.7219245657325, 'timeline_first_chunk_emitted_delta_ms': 0.0017024576663970947, 'timeline_llm_done_ms': 3558.602520264685, 'timeline_llm_done_delta_ms': 2457.8805956989527, 'timeline_stream_done_ms': 3570.770296268165, 'timeline_stream_done_delta_ms': 12.167776003479958, 'timeline_request_done_ms': 3570.7718394696712, 'timeline_request_done_delta_ms': 0.001543201506137848, 'timeline_to_first_batch_ms': 52.93853487819433, 'timeline_to_first_chunk_emitted_ms': 1100.7219245657325, 'timeline_first_batch_to_first_chunk_emitted_ms': 1047.7833896875381, 'timeline_first_chunk_to_request_done_ms': 2470.0499149039388, 'llm_first_token_ms': 46.51665687561035, 'llm_queued_to_scheduled_ms': 0.014347024261951447, 'llm_time_in_queue_ms': 0.014347024261951447, 'llm_scheduled_to_first_token_ms': 30.961014330387115, 'llm_scheduler_ms': 30.961014330387115, 'llm_first_to_last_token_ms': 2000.2922853454947, 'llm_model_execute_ms': 2000.2922853454947, 'llm_queued_to_last_token_ms': 2031.2676467001438, 'llm_request_lifecycle_ms': 2031.2676467001438}}, {'success': True, 'latency_ms': 3800.116630271077, 'ttfb_ms': 1136.2931989133358, 'chunks_sent': 5, 'total_bytes': 120364, 'audio_seconds': 3.76, 'speaker': 'krishna', 'error': None, 'inter_chunk_ms': [554.2768985033035, 1915.8302629366517, 12.340252287685871, 0.003125518560409546], 'tail_emit_speed_x': 1.4115049652835066, 'final_metrics': {'ttfb_ms': 1100.486308336258, 'chunks_sent': 5, 'total_bytes': 120364, 'audio_duration_seconds': 3.76, 'text_chunked': False, 'prompt_build_ms': 0.006943009793758392, 'sampling_params_ms': 0.007383525371551514, 'llm_generation_wall_ms': 3570.6742955371737, 'llm_batch_count': 92, 'llm_token_total': 223, 'semantic_token_total': 188, 'global_token_total': 32, 'llm_parse_ms': 0.24444516748189926, 'bicodec_decode_calls': 4, 'llm_decode_calls': 4, 'stream_output_kind': 'delta', 'stream_decode_interval_tokens': 48, 'stream_window_size_tokens': 128, 'stream_windowed_decode': True, 'timeline_markers': [{'stage': 'request_start', 't_ms': 0.0, 'dt_ms': 0.0}, {'stage': 'prompt_ready', 't_ms': 0.008235685527324677, 'dt_ms': 0.008235685527324677}, {'stage': 'sampling_ready', 't_ms': 0.01666136085987091, 'dt_ms': 0.008425675332546234}, {'stage': 'llm_first_batch', 't_ms': 52.61671543121338, 'dt_ms': 52.60005407035351}, {'stage': 'first_chunk_ready', 't_ms': 367.77163948863745, 'dt_ms': 315.15492405742407}, {'stage': 'first_chunk_decoded', 't_ms': 1100.4847763106227, 'dt_ms': 732.7131368219852}, {'stage': 'first_chunk_emitted', 't_ms': 1100.4864489659667, 'dt_ms': 0.0016726553440093994}, {'stage': 'llm_done', 't_ms': 3570.7017658278346, 'dt_ms': 2470.215316861868}, {'stage': 'stream_done', 't_ms': 3582.9522786661983, 'dt_ms': 12.250512838363647}, {'stage': 'request_done', 't_ms': 3582.954061217606, 'dt_ms': 0.0017825514078140259}], 'timeline_total_ms': 3582.9636389389634, 'enabled': 1.0, 'batch_total_batches': 69.0, 'batch_total_requests': 623.0, 'batch_avg_size': 9.028985507246377, 'batch_max_seen': 16.0, 'llm_batch_wall_ms_total': 3570.5887330695987, 'llm_batch_wall_ms_min': 3.328174352645874, 'llm_batch_wall_ms_max': 1786.2806860357523, 'llm_batch_wall_ms_p50': 7.026090286672115, 'llm_decode_wall_ms_total': 3518.000620417297, 'llm_decode_wall_ms_min': 3.328174352645874, 'llm_decode_wall_ms_max': 1786.2806860357523, 'llm_decode_wall_ms_p50': 7.026090286672115, 'tokens_per_batch_total': 223.0, 'tokens_per_batch_min': 1.0, 'tokens_per_batch_max': 47.0, 'tokens_per_batch_p50': 1.0, 'llm_parse_step_ms_total': 0.24444516748189926, 'llm_parse_step_ms_min': 0.0005504116415977478, 'llm_parse_step_ms_max': 0.047089532017707825, 'llm_parse_step_ms_p50': 0.001232139766216278, 'bicodec_decode_wall_ms_total': 2810.77959574759, 'bicodec_decode_wall_ms_min': 12.16943934559822, 'bicodec_decode_wall_ms_max': 1786.1907575279474, 'bicodec_decode_wall_ms_p50': 465.74350632727146, 'bicodec_decode_tokens_total': 460.0, 'bicodec_decode_tokens_min': 48.0, 'bicodec_decode_tokens_max': 188.0, 'bicodec_decode_tokens_p50': 96.0, 'bicodec_decode_interval_ms_total': 2482.41448123008, 'bicodec_decode_interval_ms_min': 12.33210600912571, 'bicodec_decode_interval_ms_max': 1915.8233301714063, 'bicodec_decode_interval_ms_p50': 554.2590450495481, 'bicodec_decode_wall_ms': 2810.77959574759, 'bicodec_decode_gpu_ms': 0.0, 'bicodec_decode_cpu_ms': 2810.77959574759, 'llm_time_per_token_ms': 16.01199235666894, 'llm_time_per_batch_wall_ms': 38.810747098582596, 'timeline_request_start_ms': 0.0, 'timeline_request_start_delta_ms': 0.0, 'timeline_prompt_ready_ms': 0.008235685527324677, 'timeline_prompt_ready_delta_ms': 0.008235685527324677, 'timeline_sampling_ready_ms': 0.01666136085987091, 'timeline_sampling_ready_delta_ms': 0.008425675332546234, 'timeline_llm_first_batch_ms': 52.61671543121338, 'timeline_llm_first_batch_delta_ms': 52.60005407035351, 'timeline_first_chunk_ready_ms': 367.77163948863745, 'timeline_first_chunk_ready_delta_ms': 315.15492405742407, 'timeline_first_chunk_decoded_ms': 1100.4847763106227, 'timeline_first_chunk_decoded_delta_ms': 732.7131368219852, 'timeline_first_chunk_emitted_ms': 1100.4864489659667, 'timeline_first_chunk_emitted_delta_ms': 0.0016726553440093994, 'timeline_llm_done_ms': 3570.7017658278346, 'timeline_llm_done_delta_ms': 2470.215316861868, 'timeline_stream_done_ms': 3582.9522786661983, 'timeline_stream_done_delta_ms': 12.250512838363647, 'timeline_request_done_ms': 3582.954061217606, 'timeline_request_done_delta_ms': 0.0017825514078140259, 'timeline_to_first_batch_ms': 52.61671543121338, 'timeline_to_first_chunk_emitted_ms': 1100.4864489659667, 'timeline_first_batch_to_first_chunk_emitted_ms': 1047.8697335347533, 'timeline_first_chunk_to_request_done_ms': 2482.4676122516394, 'llm_first_token_ms': 46.167850494384766, 'llm_queued_to_scheduled_ms': 0.010710209608078003, 'llm_time_in_queue_ms': 0.010710209608078003, 'llm_scheduled_to_first_token_ms': 30.961014330387115, 'llm_scheduler_ms': 30.961014330387115, 'llm_first_to_last_token_ms': 2130.3564347326756, 'llm_model_execute_ms': 2130.3564347326756, 'llm_queued_to_last_token_ms': 2161.3281592726707, 'llm_request_lifecycle_ms': 2161.3281592726707}}, {'success': True, 'latency_ms': 3953.108476474881, 'ttfb_ms': 1136.3817155361176, 'chunks_sent': 5, 'total_bytes': 119724, 'audio_seconds': 3.74, 'speaker': 'anika', 'error': None, 'inter_chunk_ms': [554.2954141274095, 1928.2354675233364, 333.91964714974165, 0.0098496675491333], 'tail_emit_speed_x': 1.3277823223270426, 'final_metrics': {'ttfb_ms': 1100.2429081127048, 'chunks_sent': 5, 'total_bytes': 119724, 'audio_duration_seconds': 3.74, 'text_chunked': False, 'prompt_build_ms': 0.0070231035351753235, 'sampling_params_ms': 0.007032416760921478, 'llm_generation_wall_ms': 3916.6885502636433, 'llm_batch_count': 92, 'llm_token_total': 245, 'semantic_token_total': 210, 'global_token_total': 32, 'llm_parse_ms': 0.2947952598333359, 'bicodec_decode_calls': 4, 'llm_decode_calls': 4, 'stream_output_kind': 'delta', 'stream_decode_interval_tokens': 48, 'stream_window_size_tokens': 128, 'stream_windowed_decode': True, 'timeline_markers': [{'stage': 'request_start', 't_ms': 0.0, 'dt_ms': 0.0}, {'stage': 'prompt_ready', 't_ms': 0.008336268365383148, 'dt_ms': 0.008336268365383148}, {'stage': 'sampling_ready', 't_ms': 0.016440637409687042, 'dt_ms': 0.008104369044303894}, {'stage': 'llm_first_batch', 't_ms': 52.29828413575888, 'dt_ms': 52.28184349834919}, {'stage': 'first_chunk_ready', 't_ms': 367.44669545441866, 'dt_ms': 315.1484113186598}, {'stage': 'first_chunk_decoded', 't_ms': 1100.2414850518107, 'dt_ms': 732.7947895973921}, {'stage': 'first_chunk_emitted', 't_ms': 1100.243047811091, 'dt_ms': 0.001562759280204773}, {'stage': 'llm_done', 't_ms': 3916.7145676910877, 'dt_ms': 2816.471519879997}, {'stage': 'stream_done', 't_ms': 3916.7183758690953, 'dt_ms': 0.0038081780076026917}, {'stage': 'request_done', 't_ms': 3916.7189272120595, 'dt_ms': 0.0005513429641723633}], 'timeline_total_ms': 3916.7276229709387, 'enabled': 1.0, 'batch_total_batches': 72.0, 'batch_total_requests': 650.0, 'batch_avg_size': 9.027777777777779, 'batch_max_seen': 16.0, 'llm_batch_wall_ms_total': 3582.7685510739684, 'llm_batch_wall_ms_min': 3.3277347683906555, 'llm_batch_wall_ms_max': 1798.7694600597024, 'llm_batch_wall_ms_p50': 7.024687714874744, 'llm_decode_wall_ms_total': 3530.4980305954814, 'llm_decode_wall_ms_min': 3.3277347683906555, 'llm_decode_wall_ms_max': 1798.7694600597024, 'llm_decode_wall_ms_p50': 7.024687714874744, 'tokens_per_batch_total': 245.0, 'tokens_per_batch_min': 1.0, 'tokens_per_batch_max': 61.0, 'tokens_per_batch_p50': 1.0, 'llm_parse_step_ms_total': 0.2947952598333359, 'llm_parse_step_ms_min': 0.0005010515451431274, 'llm_parse_step_ms_max': 0.0713542103767395, 'llm_parse_step_ms_p50': 0.0014221295714378357, 'bicodec_decode_wall_ms_total': 3145.011892542243, 'bicodec_decode_wall_ms_min': 333.75544659793377, 'bicodec_decode_wall_ms_max': 1798.6691305413842, 'bicodec_decode_wall_ms_p50': 465.82066267728806, 'bicodec_decode_tokens_total': 400.0, 'bicodec_decode_tokens_min': 48.0, 'bicodec_decode_tokens_max': 128.0, 'bicodec_decode_tokens_p50': 96.0, 'bicodec_decode_interval_ms_total': 2816.4332676678896, 'bicodec_decode_interval_ms_min': 333.92452728003263, 'bicodec_decode_interval_ms_max': 1928.2327741384506, 'bicodec_decode_interval_ms_p50': 554.2759662494063, 'bicodec_decode_wall_ms': 3145.011892542243, 'bicodec_decode_gpu_ms': 0.0, 'bicodec_decode_cpu_ms': 3145.011892542243, 'llm_time_per_token_ms': 15.986483878627116, 'llm_time_per_batch_wall_ms': 38.943136424717046, 'timeline_request_start_ms': 0.0, 'timeline_request_start_delta_ms': 0.0, 'timeline_prompt_ready_ms': 0.008336268365383148, 'timeline_prompt_ready_delta_ms': 0.008336268365383148, 'timeline_sampling_ready_ms': 0.016440637409687042, 'timeline_sampling_ready_delta_ms': 0.008104369044303894, 'timeline_llm_first_batch_ms': 52.29828413575888, 'timeline_llm_first_batch_delta_ms': 52.28184349834919, 'timeline_first_chunk_ready_ms': 367.44669545441866, 'timeline_first_chunk_ready_delta_ms': 315.1484113186598, 'timeline_first_chunk_decoded_ms': 1100.2414850518107, 'timeline_first_chunk_decoded_delta_ms': 732.7947895973921, 'timeline_first_chunk_emitted_ms': 1100.243047811091, 'timeline_first_chunk_emitted_delta_ms': 0.001562759280204773, 'timeline_llm_done_ms': 3916.7145676910877, 'timeline_llm_done_delta_ms': 2816.471519879997, 'timeline_stream_done_ms': 3916.7183758690953, 'timeline_stream_done_delta_ms': 0.0038081780076026917, 'timeline_request_done_ms': 3916.7189272120595, 'timeline_request_done_delta_ms': 0.0005513429641723633, 'timeline_to_first_batch_ms': 52.29828413575888, 'timeline_to_first_chunk_emitted_ms': 1100.243047811091, 'timeline_first_batch_to_first_chunk_emitted_ms': 1047.944763675332, 'timeline_first_chunk_to_request_done_ms': 2816.4758794009686, 'llm_first_token_ms': 45.82357406616211, 'llm_queued_to_scheduled_ms': 0.006161630153656006, 'llm_time_in_queue_ms': 0.006161630153656006, 'llm_scheduled_to_first_token_ms': 30.961014330387115, 'llm_scheduler_ms': 30.961014330387115, 'llm_first_to_last_token_ms': 2307.7314840629697, 'llm_model_execute_ms': 2307.7314840629697, 'llm_queued_to_last_token_ms': 2338.6986600235105, 'llm_request_lifecycle_ms': 2338.6986600235105}}]}]}


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Refining parsing focus to levels**


python3 - <<'PY'
import json,os
p='/tmp/stream_levels_with_bicodec_batch.json'
d=json.load(open(p))
levels=d.get('levels',[])
print('n_levels',len(levels))
for r in levels:
    c=r.get('concurrency')
    t=r.get('throughput_rps',0)
    lat=r.get('latency_ms',{})
    ttfb=r.get('ttfb_ms',{})
    tim=r.get('timing',{})
    gpu=r.get('gpu',{})
    util=(gpu.get('gpu_util_pct') or {})
    print(f"c={c} thr={t:.2f} lat={lat.get('avg',0):.1f} ttfb={ttfb.get('avg',0):.1f} llm_tok_ms={tim.get('llm_time_per_token_ms',{}).get('avg',0):.2f} bicodec_total={tim.get('bicodec_decode_wall_ms_total',{}).get('avg',0):.1f} batch_avg={tim.get('batch_avg_size',{}).get('avg',0):.2f} batch_max={tim.get('batch_max_seen',{}).get('avg',0):.1f} util={util.get('avg',0):.1f}")
PY


Output:
Chunk ID: c6da90
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 89
Output:
n_levels 3
c=32 thr=15.84 lat=1833.6 ttfb=648.9 llm_tok_ms=7.72 bicodec_total=623.9 batch_avg=12.16 batch_max=16.0 util=65.8
c=64 thr=22.22 lat=2530.5 ttfb=686.9 llm_tok_ms=11.06 bicodec_total=1396.7 batch_avg=10.58 batch_max=16.0 util=74.6
c=96 thr=24.26 lat=3225.0 ttfb=927.7 llm_tok_ms=13.96 bicodec_total=1905.3 batch_avg=9.59 batch_max=16.0 util=74.7


Summary: auto


**Planning baseline comparison**


python3 - <<'PY'
import json,glob,os
cand=[*glob.glob('**/stream_baseline_levels_detailed.json',recursive=True),'/tmp/stream_baseline_levels_detailed.json']
seen=[]
for p in cand:
    if os.path.exists(p) and p not in seen:
        seen.append(p)
if not seen:
    print('none')
    raise SystemExit
for p in seen:
    d=json.load(open(p))
    levels=d.get('levels',d.get('results',[]))
    print('\n',p,'n_levels',len(levels))
    for r in levels:
        c=r.get('concurrency')
        t=r.get('throughput_rps',0)
        lat=r.get('latency_ms',{})
        ttfb=r.get('ttfb_ms',{})
        tim=r.get('timing',{})
        gpu=r.get('gpu',{})
        util=(gpu.get('gpu_util_pct') or {})
        print(f"c={c} thr={t:.2f} lat={lat.get('avg',0):.1f} ttfb={ttfb.get('avg',0):.1f} llm_tok_ms={tim.get('llm_time_per_token_ms',{}).get('avg',0):.2f} bicodec_total={tim.get('bicodec_decode_wall_ms_total',{}).get('avg',0):.1f} batch_avg={tim.get('batch_avg_size',{}).get('avg',0):.2f} util={util.get('avg',0):.1f}")
PY


Output:
Chunk ID: 6e306f
Wall time: 0.3225 seconds
Process exited with code 0
Original token count: 90
Output:

 /tmp/stream_baseline_levels_detailed.json n_levels 3
c=32 thr=12.56 lat=2546.5 ttfb=1046.7 llm_tok_ms=10.62 bicodec_total=1943.5 batch_avg=0.00 util=66.8
c=64 thr=16.77 lat=3811.9 ttfb=1378.0 llm_tok_ms=16.55 bicodec_total=3147.9 batch_avg=0.00 util=66.7
c=96 thr=18.92 lat=5069.5 ttfb=1979.0 llm_tok_ms=22.09 bicodec_total=4293.1 batch_avg=0.00 util=64.3


**Planning adaptive decode batching**


The new 3-level run finished and confirms large gains with decoder micro-batching. Next I’m inspecting runtime knobs in `streaming_pipeline.py` so we can add/load-adaptive decode scheduling and then benchmark a tuning grid for peak throughput vs TTFB.


**Inspecting decode interval and scheduling**


sed -n '1,260p' veena3modal/core/streaming_pipeline.py


sed -n '260,620p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: b41261
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 1716
Output:
veena3modal/core/bicodec_decoder.py:78:    use_sliding_window: bool
veena3modal/core/bicodec_decoder.py:91:    Supports streaming with sliding window approach (like SNAC).
veena3modal/core/bicodec_decoder.py:279:            return [self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)]
veena3modal/core/bicodec_decoder.py:281:        # Sliding window path uses special trimming behavior; keep it sequential.
veena3modal/core/bicodec_decoder.py:284:                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
veena3modal/core/bicodec_decoder.py:291:                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
veena3modal/core/bicodec_decoder.py:344:                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
veena3modal/core/bicodec_decoder.py:381:                    (req.semantic_ids, req.global_ids, req.use_sliding_window, req.trim_warmup)
veena3modal/core/bicodec_decoder.py:468:        use_sliding_window: bool = False,
veena3modal/core/bicodec_decoder.py:485:            use_sliding_window: If True, return only middle samples
veena3modal/core/bicodec_decoder.py:518:        # If using sliding window, return only the middle portion
veena3modal/core/bicodec_decoder.py:519:        if use_sliding_window and len(audio) > 4096:
veena3modal/core/bicodec_decoder.py:539:        use_sliding_window: bool = False,
veena3modal/core/bicodec_decoder.py:552:            use_sliding_window: Use sliding window mode
veena3modal/core/bicodec_decoder.py:558:        if self.enable_batching and not use_sliding_window:
veena3modal/core/bicodec_decoder.py:564:                use_sliding_window=use_sliding_window,
veena3modal/core/bicodec_decoder.py:577:            use_sliding_window,
veena3modal/core/streaming_pipeline.py:10:4. Apply sliding window (every N token pairs)
veena3modal/core/streaming_pipeline.py:20:2. Apply sliding window
veena3modal/core/streaming_pipeline.py:80:    Streaming TTS pipeline using sliding window approach.
veena3modal/core/streaming_pipeline.py:83:    - Decoding overlapping 28-token windows (4 frames)
veena3modal/core/streaming_pipeline.py:97:        Initialize sliding window streaming pipeline.
veena3modal/core/streaming_pipeline.py:115:        print(f"🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)")
veena3modal/core/streaming_pipeline.py:191:        Generate speech audio with sliding window streaming.
veena3modal/core/streaming_pipeline.py:193:        Yields audio chunks using overlapping windows for smooth playback.
veena3modal/core/streaming_pipeline.py:206:        logger.debug("sliding-window streaming generation started")
veena3modal/core/streaming_pipeline.py:225:        logger.debug("sampling temp=%s top_p=%s sliding_window=28", temperature, top_p)
veena3modal/core/streaming_pipeline.py:245:        # Stream tokens with sliding window
veena3modal/core/streaming_pipeline.py:259:                    # Process every 7 tokens (1 frame) when we have enough for sliding window
veena3modal/core/streaming_pipeline.py:262:                        # Sliding window: take last 28 tokens (4 frames)
veena3modal/core/streaming_pipeline.py:263:                        window_tokens = token_buffer[-28:]
veena3modal/core/streaming_pipeline.py:265:                        # Decode with sliding window mode (returns middle 2048 samples only)
veena3modal/core/streaming_pipeline.py:268:                                window_tokens,
veena3modal/core/streaming_pipeline.py:269:                                trim_warmup=False,  # Sliding window handles trimming
veena3modal/core/streaming_pipeline.py:270:                                use_sliding_window=True  # CRITICAL: Use sliding window mode
veena3modal/core/streaming_pipeline.py:274:                                window_tokens,
veena3modal/core/streaming_pipeline.py:276:                                use_sliding_window=True
veena3modal/core/streaming_pipeline.py:282:                                logger.debug("first sliding-window chunk decoded bytes=%d", len(audio_bytes))
veena3modal/core/streaming_pipeline.py:285:        # Note: No final chunk processing needed - sliding window handles all tokens
veena3modal/core/streaming_pipeline.py:289:            "sliding-window complete tokens=%d chunks=%d",
veena3modal/core/streaming_pipeline.py:325:        Generate speech audio with sliding window streaming for Indic model.
veena3modal/core/streaming_pipeline.py:327:        Yields audio chunks using overlapping windows for smooth playback.
veena3modal/core/streaming_pipeline.py:419:        bicodec_decode_intervals_ms: List[float] = []
veena3modal/core/streaming_pipeline.py:503:                            use_sliding_window=False,
veena3modal/core/streaming_pipeline.py:511:                            bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
veena3modal/core/streaming_pipeline.py:563:                use_sliding_window=False,
veena3modal/core/streaming_pipeline.py:571:                bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
veena3modal/core/streaming_pipeline.py:628:            "stream_decode_interval_tokens": int(_STREAM_DECODE_INTERVAL),
veena3modal/core/streaming_pipeline.py:629:            "stream_window_size_tokens": int(_STREAM_WINDOW_SIZE),
veena3modal/core/streaming_pipeline.py:630:            "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
veena3modal/core/streaming_pipeline.py:652:        self._apply_stats(perf, "bicodec_decode_interval_ms", bicodec_decode_intervals_ms)
veena3modal/core/streaming_pipeline.py:855:                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]
veena3modal/core/streaming_pipeline.py:860:                                global_ids=window_global,
veena3modal/core/streaming_pipeline.py:862:                                use_sliding_window=False
veena3modal/core/streaming_pipeline.py:867:                                global_ids=window_global,
veena3modal/core/streaming_pipeline.py:868:                                use_sliding_window=False,
veena3modal/core/streaming_pipeline.py:899:                use_sliding_window=False,
veena3modal/core/streaming_pipeline.py:1078:                            use_sliding_window=False
veena3modal/core/streaming_pipeline.py:1084:                            use_sliding_window=False,
veena3modal/core/streaming_pipeline.py:1129:                use_sliding_window=False,
veena3modal/core/streaming_pipeline.py:1192:                "stream_decode_interval_tokens": int(DECODE_INTERVAL),
veena3modal/core/streaming_pipeline.py:1193:                "stream_window_size_tokens": int(WINDOW_SIZE),
veena3modal/core/streaming_pipeline.py:1194:                "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),


Output:
Chunk ID: ee2006
Wall time: 0.0506 seconds
Process exited with code 0
Original token count: 2232
Output:
"""
Veena3 Streaming Pipeline - Sliding Window Approach

Implements streaming for both SNAC and BiCodec tokens.

For BiCodec (Spark TTS):
1. Generate tokens from vLLM
2. Parse NEW token IDs incrementally (O(1) per token via cache)
3. Buffer semantic + global tokens
4. Apply sliding window (every N token pairs)
5. Decode and stream audio chunks

OPTIMIZATION (Dec 2025):
- Replaced O(n²) pattern (decode-all + regex-all per iteration)
- Now uses incremental token parsing with BiCodecTokenParser
- ~10x CPU reduction in streaming hot loop

For SNAC (legacy):
1. Filter SNAC token IDs directly
2. Apply sliding window
3. Decode and stream
"""

import asyncio
import logging
import os
import re
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union
from vllm import SamplingParams
from vllm.sampling_params import RequestOutputKind
from veena3modal.audio.crossfade import crossfade_bytes_int16
from veena3modal.core.token_utils import BiCodecTokenParser

from veena3modal.core.constants import (
    CODE_END_TOKEN_ID,
    CODE_START_TOKEN_ID,
    SNAC_MIN_ID,
    SNAC_MAX_ID,
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_MIN_TOKENS,
    DEFAULT_REPETITION_PENALTY,
    DEFAULT_SEED,
)

logger = logging.getLogger(__name__)
_STREAM_OUTPUT_KIND_RAW = os.environ.get("VEENA3_STREAM_OUTPUT_KIND", "delta").strip().lower()
_STREAM_USE_DELTA_OUTPUT = _STREAM_OUTPUT_KIND_RAW in {"delta", "d", "1", "true", "yes", "on"}

def _env_int(name: str, default: int, minimum: int = 0) -> int:
    raw = os.environ.get(name, "")
    if not raw:
        return default
    try:
        value = int(raw)
    except ValueError:
        return default
    return max(minimum, value)

_STREAM_EXPECTED_GLOBAL_COUNT = 32
_STREAM_MIN_SEMANTIC_FIRST_CHUNK = _env_int("VEENA3_STREAM_MIN_SEMANTIC_FIRST", 10, minimum=1)
_STREAM_DECODE_INTERVAL = _env_int("VEENA3_STREAM_DECODE_INTERVAL", 48, minimum=1)
_STREAM_CROSSFADE_MS = _env_int("VEENA3_STREAM_CROSSFADE_MS", 50, minimum=0)
_STREAM_WINDOW_SIZE = _env_int("VEENA3_STREAM_WINDOW_SIZE", 128, minimum=16)
_STREAM_WINDOWED_DECODE = os.environ.get("VEENA3_STREAM_WINDOWED_DECODE", "true").strip().lower() in {
    "1",
    "true",
    "yes",
    "on",
}

class Veena3SlidingWindowPipeline:
    """
    Streaming TTS pipeline using sliding window approach.

    This eliminates choppy audio and popping artifacts by:
    - Decoding overlapping 28-token windows (4 frames)
    - Keeping only the middle 2048 samples from each decode
    - Creating natural continuity between chunks

    Based on the official Canopy Labs implementation.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        snac_decoder,
    ):
        """
        Initialize sliding window streaming pipeline.

        Args:
            model: Veena3Model instance
            prompt_builder: Veena3PromptBuilder instance
            snac_decoder: SNACDecoder instance (with batching enabled)
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.snac_decoder = snac_decoder

        # OPTIMIZATION: Pre-warm BiCodecTokenParser once at init, not per-request
        # Saves ~123ms per streaming TTFB by avoiding repeated 166K-entry vocab iteration
        tokenizer = getattr(model, "tokenizer", None)
        if tokenizer is None:
            tokenizer = getattr(model.engine, "tokenizer", None)
        self.token_parser = BiCodecTokenParser(tokenizer) if tokenizer else None

        print(f"🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)")

    @staticmethod
    def _p50(values: List[float]) -> float:
        if not values:
            return 0.0
        ordered = sorted(values)
        return ordered[(len(ordered) - 1) // 2]

    @staticmethod
    def _to_number(value: Any) -> Optional[float]:
        if value is None:
            return None
        if isinstance(value, (int, float)):
            return float(value)
        try:
            return float(value)
        except (TypeError, ValueError):
            return None

    def _extract_request_metrics(self, raw_metrics: Any) -> Dict[str, float]:
        if raw_metrics is None:
            return {}

        numeric: Dict[str, float] = {}
        items: List[Tuple[str, Any]] = []

        if isinstance(raw_metrics, dict):
            items.extend(raw_metrics.items())
        else:
            if hasattr(raw_metrics, "__dict__"):
                try:
                    items.extend(dict(raw_metrics.__dict__).items())
                except Exception:
                    pass
            for name in dir(raw_metrics):
                if name.startswith("_"):
                    continue
                try:
                    value = getattr(raw_metrics, name)
                except Exception:
                    continue
                if callable(value):
                    continue
                items.append((name, value))

        seen: set[str] = set()
        for key, value in items:
            key_s = str(key)
            if key_s in seen:
                continue
            seen.add(key_s)
            parsed = self._to_number(value)
            if parsed is not None:
                numeric[key_s] = parsed
        return numeric

    def _apply_stats(self, perf: Dict[str, Any], prefix: str, values: List[float]) -> None:
        if not values:
            return
        perf[f"{prefix}_total"] = float(sum(values))
        perf[f"{prefix}_min"] = float(min(values))
        perf[f"{prefix}_max"] = float(max(values))
        perf[f"{prefix}_p50"] = float(self._p50(values))

    async def generate_speech_stream(
        self,
        description: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
    ) -> AsyncGenerator[bytes, None]:
        """
        Generate speech audio with sliding window streaming.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            description: Character/voice description
            text: Text to synthesize (with optional <emotion> tags)
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops

        Yields:
            Audio bytes (int16 PCM, 24kHz mono)
        """
        logger.debug("sliding-window streaming generation started")
        logger.debug("description_prefix=%s", description[:80])
        logger.debug("text_len=%d", len(text))

        # Build prompt
        prompt = self.prompt_builder.build_prefix(description, text)
        logger.debug("prompt built chars=%d", len(prompt))

        # Configure sampling
        sampling_params = SamplingParams(
            temperature=temperature,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop_token_ids=[CODE_END_TOKEN_ID],  # Only audio EOS
            seed=seed,  # None = random, int = reproducible
        )

        logger.debug("sampling temp=%s top_p=%s sliding_window=28", temperature, top_p)

        # Token buffer - keeps ALL tokens (not chunked)
        token_buffer = []
        total_tokens_generated = 0
        total_audio_chunks = 0

        # Generate tokens with vLLM (streaming)
        print(f"🔮 Starting token generation...")

        # Generate unique request ID for concurrent streaming support
        import uuid
        import time
        request_id = f"slide-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Stream tokens with sliding window
        async for request_output in results_generator:
            # Extract generated token IDs
            generated_ids = request_output.outputs[0].token_ids

            # Process only new tokens
            new_tokens = generated_ids[total_tokens_generated:]
            total_tokens_generated = len(generated_ids)

            # Filter and buffer SNAC tokens
            for token_id in new_tokens:
                if SNAC_MIN_ID <= token_id <= SNAC_MAX_ID:
                    token_buffer.append(token_id)

                    # Process every 7 tokens (1 frame) when we have enough for sliding window
                    # Official approach: once count > 27, take last 28 tokens


Output:
Chunk ID: 49b822
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 4259
Output:
                    # Official approach: once count > 27, take last 28 tokens
                    if len(token_buffer) % 7 == 0 and len(token_buffer) > 27:
                        # Sliding window: take last 28 tokens (4 frames)
                        window_tokens = token_buffer[-28:]

                        # Decode with sliding window mode (returns middle 2048 samples only)
                        if self.snac_decoder.enable_batching:
                            audio_bytes = await self.snac_decoder.decode_single_async(
                                window_tokens,
                                trim_warmup=False,  # Sliding window handles trimming
                                use_sliding_window=True  # CRITICAL: Use sliding window mode
                            )
                        else:
                            audio_bytes = self.snac_decoder.decode_to_bytes(
                                window_tokens,
                                trim_warmup=False,
                                use_sliding_window=True
                            )

                        if audio_bytes:
                            total_audio_chunks += 1
                            if total_audio_chunks == 1:
                                logger.debug("first sliding-window chunk decoded bytes=%d", len(audio_bytes))
                            yield audio_bytes

        # Note: No final chunk processing needed - sliding window handles all tokens
        # as they come in (every 7 tokens after the first 28)

        logger.debug(
            "sliding-window complete tokens=%d chunks=%d",
            total_tokens_generated,
            total_audio_chunks,
        )

    def _extract_bicodec_tokens_from_text(self, text: str) -> tuple[List[int], List[int]]:
        """
        Extract BiCodec semantic and global tokens from generated text.

        Args:
            text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
        """
        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", text)
        semantic_ids = [int(t) for t in semantic_matches]

        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", text)
        global_ids = [int(t) for t in global_matches]

        return semantic_ids, global_ids

    async def generate_speech_stream_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,  # Added for Spark TTS parity with non-streaming
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
        seed: Optional[int] = None,
        emit_progress: bool = False,
    ) -> AsyncGenerator[Union[bytes, Tuple[bytes, Dict[str, Any]]], None]:
        """
        Generate speech audio with sliding window streaming for Indic model.

        Yields audio chunks using overlapping windows for smooth playback.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "<laugh> Hello there!"
                - "नमस्ते! <excited> आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_p: Nucleus sampling
            max_tokens: Max SNAC tokens to generate
            repetition_penalty: Prevent loops
            seed: Random seed for reproducibility

        Yields:
            Audio bytes (int16 PCM, 16kHz mono - BiCodec)
        """
        import time
        import uuid

        t_request_start = time.perf_counter()
        timeline_marks: List[Dict[str, float]] = [{"stage": "request_start", "t_ms": 0.0, "dt_ms": 0.0}]
        last_mark_ms = 0.0
        mark_stage_set: set[str] = {"request_start"}

        def _mark(stage: str) -> float:
            nonlocal last_mark_ms
            now_ms = (time.perf_counter() - t_request_start) * 1000.0
            dt_ms = now_ms - last_mark_ms
            last_mark_ms = now_ms
            timeline_marks.append({"stage": stage, "t_ms": float(now_ms), "dt_ms": float(dt_ms)})
            mark_stage_set.add(stage)
            return float(now_ms)

        prompt_t0 = time.perf_counter()
        prompt = self.prompt_builder.build_prefix(speaker, text)
        prompt_build_ms = (time.perf_counter() - prompt_t0) * 1000.0
        _mark("prompt_ready")

        sampling_t0 = time.perf_counter()
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            min_tokens=DEFAULT_MIN_TOKENS,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
            output_kind=(RequestOutputKind.DELTA if _STREAM_USE_DELTA_OUTPUT else RequestOutputKind.CUMULATIVE),
        )
        sampling_params_ms = (time.perf_counter() - sampling_t0) * 1000.0
        _mark("sampling_ready")

        request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
        token_parser = self.token_parser
        if token_parser is None:
            tokenizer = getattr(self.model, "tokenizer", None) or getattr(self.model.engine, "tokenizer", None)
            token_parser = BiCodecTokenParser(tokenizer) if tokenizer else None
        if token_parser is None:
            raise RuntimeError("BiCodecTokenParser unavailable for streaming")

        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        semantic_buffer: List[int] = []
        global_buffer: List[int] = []
        processed_token_count = 0
        total_generated_tokens = 0
        total_audio_chunks = 0
        total_samples_emitted_to_user = 0
        previous_chunk_tail: Optional[bytes] = None
        last_decode_count = 0
        last_decode_end_ts: Optional[float] = None
        t_first_chunk_decode_done: Optional[float] = None
        t_first_chunk_emitted: Optional[float] = None
        final_request_metrics: Dict[str, float] = {}
        prev_gpu_cumulative_s: Optional[float] = None

        llm_batch_wall_ms: List[float] = []
        llm_batch_gpu_ms: List[float] = []
        llm_decode_wall_ms: List[float] = []
        llm_decode_gpu_ms: List[float] = []
        llm_parse_ms_samples: List[float] = []
        tokens_per_batch: List[float] = []
        bicodec_decode_wall_ms: List[float] = []
        bicodec_decode_tokens: List[float] = []
        bicodec_decode_intervals_ms: List[float] = []

        t_llm_start = time.perf_counter()
        t_last_batch = t_llm_start

        async for request_output in results_generator:
            now = time.perf_counter()
            batch_wall = (now - t_last_batch) * 1000.0
            t_last_batch = now

            outputs = request_output.outputs or []
            if not outputs:
                continue

            generated_ids = outputs[0].token_ids
            if _STREAM_USE_DELTA_OUTPUT:
                new_token_ids = generated_ids
                processed_token_count += len(new_token_ids)
            else:
                new_token_ids = generated_ids[processed_token_count:]
                processed_token_count = len(generated_ids)

            new_token_count = len(new_token_ids)
            if new_token_count <= 0:
                continue

            total_generated_tokens += new_token_count
            llm_batch_wall_ms.append(float(batch_wall))
            tokens_per_batch.append(float(new_token_count))

            if "llm_first_batch" not in mark_stage_set:
                _mark("llm_first_batch")
            elif llm_batch_wall_ms:
                llm_decode_wall_ms.append(float(batch_wall))

            raw_req_metrics = self._extract_request_metrics(getattr(request_output, "metrics", None))
            if raw_req_metrics:
                final_request_metrics = raw_req_metrics
                cumulative_gpu_s: Optional[float] = None
                if "model_execute_time" in raw_req_metrics:
                    cumulative_gpu_s = raw_req_metrics["model_execute_time"]
                elif "model_forward_time" in raw_req_metrics:
                    cumulative_gpu_s = raw_req_metrics["model_forward_time"]

                if cumulative_gpu_s is not None:
                    if prev_gpu_cumulative_s is None:
                        gpu_delta_ms = cumulative_gpu_s * 1000.0
                    elif cumulative_gpu_s >= prev_gpu_cumulative_s:
                        gpu_delta_ms = (cumulative_gpu_s - prev_gpu_cumulative_s) * 1000.0
                    else:
                        gpu_delta_ms = cumulative_gpu_s * 1000.0
                    prev_gpu_cumulative_s = cumulative_gpu_s
                    if gpu_delta_ms >= 0.0:
                        llm_batch_gpu_ms.append(float(gpu_delta_ms))
                        if len(llm_batch_wall_ms) > 1:
                            llm_decode_gpu_ms.append(float(gpu_delta_ms))

            parse_t0 = time.perf_counter()
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
            llm_parse_ms_samples.append((time.perf_counter() - parse_t0) * 1000.0)

            if len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
                semantic_count = len(semantic_buffer)
                if semantic_count >= _STREAM_MIN_SEMANTIC_FIRST_CHUNK:
                    if "first_chunk_ready" not in mark_stage_set:
                        _mark("first_chunk_ready")

                    if semantic_count - last_decode_count >= _STREAM_DECODE_INTERVAL:
                        last_decode_count = semantic_count
                        decode_global = global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT]

                        if (
                            _STREAM_WINDOWED_DECODE
                            and semantic_count > _STREAM_WINDOW_SIZE
                        ):
                            decode_semantic = semantic_buffer[-_STREAM_WINDOW_SIZE:]
                        else:
                            decode_semantic = semantic_buffer

                        decode_t0 = time.perf_counter()
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=decode_semantic,
                            global_ids=decode_global,
                            trim_warmup=False,
                            use_sliding_window=False,
                        )
                        decode_t1 = time.perf_counter()

                        decode_wall_ms = (decode_t1 - decode_t0) * 1000.0
                        bicodec_decode_wall_ms.append(float(decode_wall_ms))
                        bicodec_decode_tokens.append(float(len(decode_semantic)))
                        if last_decode_end_ts is not None:
                            bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
                        last_decode_end_ts = decode_t1

                        if audio_bytes:
                            total_samples_decoded = len(audio_bytes) // 2
                            if (
                                _STREAM_WINDOWED_DECODE
                                and semantic_count > _STREAM_WINDOW_SIZE
                            ):
                                new_samples_approx = _STREAM_DECODE_INTERVAL * 320
                                if total_samples_decoded > new_samples_approx:
                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
                                else:
                                    new_bytes_start = 0
                                new_audio_bytes = audio_bytes[new_bytes_start:]
                            else:
                                if total_samples_decoded > total_samples_emitted_to_user:
                                    new_bytes_start = total_samples_emitted_to_user * 2
                                    new_audio_bytes = audio_bytes[new_bytes_start:]
                                else:
                                    new_audio_bytes = b""

                            if new_audio_bytes:
                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                    previous_chunk_tail,
                                    new_audio_bytes,
                                    sample_rate_hz=16000,
                                    crossfade_ms=_STREAM_CROSSFADE_MS,
                                )
                                if to_emit:
                                    total_audio_chunks += 1
                                    total_samples_emitted_to_user += len(to_emit) // 2
                                    if t_first_chunk_decode_done is None:
                                        t_first_chunk_decode_done = decode_t1
                                        _mark("first_chunk_decoded")
                                    if t_first_chunk_emitted is None:
                                        t_first_chunk_emitted = time.perf_counter()
                                        _mark("first_chunk_emitted")
                                    if emit_progress:
                                        yield (to_emit, {})
                                    else:
                                        yield to_emit

        llm_generation_wall_ms = (time.perf_counter() - t_llm_start) * 1000.0
        _mark("llm_done")

        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
            decode_semantic = semantic_buffer
            decode_t0 = time.perf_counter()
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=decode_semantic,
                global_ids=global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT],
                use_sliding_window=False,
                trim_warmup=False,
            )
            decode_t1 = time.perf_counter()
            decode_wall_ms = (decode_t1 - decode_t0) * 1000.0
            bicodec_decode_wall_ms.append(float(decode_wall_ms))
            bicodec_decode_tokens.append(float(len(decode_semantic)))
            if last_decode_end_ts is not None:
                bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
            last_decode_end_ts = decode_t1

            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2
                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=_STREAM_CROSSFADE_MS,
                    )
                    if to_emit:
                        total_audio_chunks += 1
                        total_samples_emitted_to_user += len(to_emit) // 2
                        if t_first_chunk_decode_done is None:
                            t_first_chunk_decode_done = decode_t1
                            _mark("first_chunk_decoded")
                        if t_first_chunk_emitted is None:
                            t_first_chunk_emitted = time.perf_counter()
                            _mark("first_chunk_emitted")
                        if emit_progress:
                            yield (to_emit, {})
                        else:
                            yield to_emit

        if previous_chunk_tail:
            total_audio_chunks += 1
            total_samples_emitted_to_user += len(previous_chunk_tail) // 2
            if t_first_chunk_emitted is None:
                t_first_chunk_emitted = time.perf_counter()
                _mark("first_chunk_emitted")
            if emit_progress:
                yield (previous_chunk_tail, {})
            else:
                yield previous_chunk_tail

        _mark("stream_done")
        _mark("request_done")

        perf: Dict[str, Any] = {
            "prompt_build_ms": float(prompt_build_ms),
            "sampling_params_ms": float(sampling_params_ms),
            "llm_generation_wall_ms": float(llm_generation_wall_ms),
            "llm_batch_count": int(len(llm_batch_wall_ms)),
            "llm_token_total": int(total_generated_tokens),
            "semantic_token_total": int(len(semantic_buffer)),
            "global_token_total": int(len(global_buffer)),


sed -n '620,980p' veena3modal/core/streaming_pipeline.py