Output:
Chunk ID: c9f423
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 2257
Output:
veena3modal/services/tts_runtime.py:312:        request_id = f"warmup-{speaker_name}-{int(time.time() * 1000)}"
veena3modal/services/tts_runtime.py:318:            request_id=request_id,
veena3modal/services/tts_runtime.py:375:    request_id: Optional[str] = None,
veena3modal/services/tts_runtime.py:404:        "request_id": request_id,
veena3modal/services/tts_runtime.py:413:    if request_id is None:
veena3modal/services/tts_runtime.py:415:        request_id = f"runtime-ns-{uuid.uuid4().hex[:10]}"
veena3modal/services/tts_runtime.py:416:        metrics["request_id"] = request_id
veena3modal/services/tts_runtime.py:428:                "request_id": request_id,
veena3modal/services/tts_runtime.py:453:                request_id=request_id,
veena3modal/services/tts_runtime.py:458:                request_id=request_id,
veena3modal/services/tts_runtime.py:604:    request_id: Optional[str] = None,
veena3modal/services/tts_runtime.py:732:    request_id: Optional[str] = None,
veena3modal/services/tts_runtime.py:794:                "request_id": request_id,
veena3modal/services/tts_runtime.py:819:                request_id,
veena3modal/services/tts_runtime.py:854:                    request_id=request_id,
veena3modal/services/tts_runtime.py:866:                    request_id=request_id,
veena3modal/services/tts_runtime.py:888:                    "request_id": request_id,
veena3modal/services/tts_runtime.py:907:    request_id: Optional[str] = None,
veena3modal/services/tts_runtime.py:940:                request_id=request_id,
veena3modal/services/tts_runtime.py:968:                    request_id=request_id,
veena3modal/services/tts_runtime.py:990:                    request_id=request_id,
veena3modal/core/pipeline.py:79:        request_id: Optional[str] = None,
veena3modal/core/pipeline.py:117:        if request_id is None:
veena3modal/core/pipeline.py:119:            request_id = f"req-{uuid.uuid4().hex[:12]}"
veena3modal/core/pipeline.py:125:            request_id=request_id,
veena3modal/core/pipeline.py:178:                        "request_id": request_id,
veena3modal/core/pipeline.py:193:                    extra={"speaker": speaker, "request_id": request_id},
veena3modal/core/pipeline.py:198:                    "request_id": request_id,
veena3modal/core/pipeline.py:221:                        "request_id": request_id,
veena3modal/core/pipeline.py:241:                    "request_id": request_id,
veena3modal/core/pipeline.py:258:                    "request_id": request_id,
veena3modal/core/pipeline.py:272:                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
veena3modal/core/pipeline.py:279:                    "request_id": request_id,
veena3modal/core/pipeline.py:289:            "request_id": request_id,
veena3modal/core/pipeline.py:332:        request_id: Optional[str] = None,
veena3modal/core/pipeline.py:369:            request_id=request_id,
veena3modal/api/websocket_handler.py:112:def create_error_message(code: str, message: str, request_id: str) -> str:
veena3modal/api/websocket_handler.py:119:            "request_id": request_id,
veena3modal/api/websocket_handler.py:125:    request_id: str,
veena3modal/api/websocket_handler.py:132:        "request_id": request_id,
veena3modal/api/websocket_handler.py:154:    request_id: str,
veena3modal/api/websocket_handler.py:160:        "request_id": request_id,
veena3modal/api/websocket_handler.py:181:    request_id = str(uuid.uuid4())
veena3modal/api/websocket_handler.py:182:    set_request_context(request_id=request_id)
veena3modal/api/websocket_handler.py:186:    logger.info(f"WebSocket connection accepted: {request_id}")
veena3modal/api/websocket_handler.py:202:                    request_id=request_id,
veena3modal/api/websocket_handler.py:215:                        request_id=request_id,
veena3modal/api/websocket_handler.py:232:                request_id=request_id,
veena3modal/api/websocket_handler.py:240:                request_id=request_id,
veena3modal/api/websocket_handler.py:260:                request_id=request_id,
veena3modal/api/websocket_handler.py:270:            request_id=request_id,
veena3modal/api/websocket_handler.py:283:                request_id=request_id,
veena3modal/api/websocket_handler.py:293:            request_id=request_id,
veena3modal/api/websocket_handler.py:318:                            logger.info(f"Client requested cancel: {request_id}")
veena3modal/api/websocket_handler.py:354:                    logger.info(f"Generation cancelled: {request_id}")
veena3modal/api/websocket_handler.py:366:                        request_id=request_id,
veena3modal/api/websocket_handler.py:396:            "request_id": request_id,
veena3modal/api/websocket_handler.py:407:            request_id=request_id,
veena3modal/api/websocket_handler.py:414:                request_id=request_id,
veena3modal/api/websocket_handler.py:437:            request_id=request_id,
veena3modal/api/websocket_handler.py:451:                request_id=request_id,
veena3modal/api/fastapi_app.py:184:        request_id = str(uuid.uuid4())
veena3modal/api/fastapi_app.py:199:                        request_id=request_id,
veena3modal/api/fastapi_app.py:201:                    headers={"X-Request-ID": request_id},
veena3modal/api/fastapi_app.py:216:                        request_id=request_id,
veena3modal/api/fastapi_app.py:218:                    headers={"X-Request-ID": request_id},
veena3modal/api/fastapi_app.py:228:                    headers["X-Request-ID"] = request_id
veena3modal/api/fastapi_app.py:234:                            request_id=request_id,
veena3modal/api/fastapi_app.py:256:                        "request_id": request_id,
veena3modal/api/fastapi_app.py:259:                headers={"X-Request-ID": request_id},
veena3modal/api/fastapi_app.py:270:                        "request_id": request_id,
veena3modal/api/fastapi_app.py:273:                headers={"X-Request-ID": request_id},
veena3modal/api/fastapi_app.py:291:        set_request_context(request_id=request_id)
veena3modal/api/fastapi_app.py:295:            request_id=request_id,
veena3modal/api/fastapi_app.py:305:            return await _handle_streaming_request(req, request_id, text, speaker, start_time)
veena3modal/api/fastapi_app.py:349:                            "request_id": request_id,
veena3modal/api/fastapi_app.py:352:                    headers={"X-Request-ID": request_id},
veena3modal/api/fastapi_app.py:368:                "X-Request-ID": request_id,
veena3modal/api/fastapi_app.py:387:                request_id=request_id,
veena3modal/api/fastapi_app.py:406:                request_id=request_id,
veena3modal/api/fastapi_app.py:476:                request_id=request_id,
veena3modal/api/fastapi_app.py:494:                        "request_id": request_id,
veena3modal/api/fastapi_app.py:497:                headers={"X-Request-ID": request_id},
veena3modal/api/fastapi_app.py:500:    async def _handle_streaming_request(req, request_id: str, text: str, speaker: str, start_time: float):
veena3modal/api/fastapi_app.py:520:                        "request_id": request_id,
veena3modal/api/fastapi_app.py:523:                headers={"X-Request-ID": request_id},
veena3modal/api/fastapi_app.py:534:                        "request_id": request_id,
veena3modal/api/fastapi_app.py:537:                headers={"X-Request-ID": request_id},
veena3modal/api/fastapi_app.py:541:        set_request_context(request_id=request_id)
veena3modal/api/fastapi_app.py:571:                            request_id=request_id,
veena3modal/api/fastapi_app.py:579:                            request_id=request_id,
veena3modal/api/fastapi_app.py:603:                    request_id=request_id,
veena3modal/api/fastapi_app.py:624:                    request_id=request_id,
veena3modal/api/fastapi_app.py:645:            "X-Request-ID": request_id,
veena3modal/core/streaming_pipeline.py:152:        request_id = f"slide-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
veena3modal/core/streaming_pipeline.py:156:            request_id=request_id,
veena3modal/core/streaming_pipeline.py:232:        request_id: Optional[str] = None,
veena3modal/core/streaming_pipeline.py:296:        if request_id is None:
veena3modal/core/streaming_pipeline.py:298:            request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
veena3modal/core/streaming_pipeline.py:314:            request_id=request_id,
veena3modal/core/streaming_pipeline.py:448:                                                "request_id": request_id,
veena3modal/core/streaming_pipeline.py:582:        request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
veena3modal/core/streaming_pipeline.py:590:            request_id=request_id,
veena3modal/core/streaming_pipeline.py:786:        request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
veena3modal/core/streaming_pipeline.py:794:            request_id=request_id,


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


cd /home/ubuntu/veenaModal && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '220,460p'


Output:
Chunk ID: 5c0ca4
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 3565
Output:
   220	        return semantic_ids, global_ids
   221
   222	    async def generate_speech_stream_indic(
   223	        self,
   224	        speaker: str,
   225	        text: str,
   226	        temperature: float = DEFAULT_TEMPERATURE,
   227	        top_k: int = DEFAULT_TOP_K,  # Added for Spark TTS parity with non-streaming
   228	        top_p: float = DEFAULT_TOP_P,
   229	        max_tokens: int = DEFAULT_MAX_TOKENS,
   230	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   231	        seed: Optional[int] = None,
   232	        request_id: Optional[str] = None,
   233	    ) -> AsyncGenerator[bytes, None]:
   234	        """
   235	        Generate speech audio with sliding window streaming for Indic model.
   236
   237	        Yields audio chunks using overlapping windows for smooth playback.
   238
   239	        Args:
   240	            speaker: Speaker name (one of 12 predefined speakers)
   241	            text: Text to synthesize with inline emotion tags
   242	                Examples:
   243	                - "Hello! Welcome."
   244	                - "<laugh> Hello there!"
   245	                - "नमस्ते! <excited> आज का दिन बहुत अच्छा है।"
   246	            temperature: Sampling temperature
   247	            top_p: Nucleus sampling
   248	            max_tokens: Max SNAC tokens to generate
   249	            repetition_penalty: Prevent loops
   250	            seed: Random seed for reproducibility
   251
   252	        Yields:
   253	            Audio bytes (int16 PCM, 16kHz mono - BiCodec)
   254	        """
   255	        import time
   256	        t_start = time.time()
   257	        emit_timing = _detailed_timing_enabled()
   258
   259	        # Build prompt using Indic prompt builder
   260	        prompt = self.prompt_builder.build_prefix(speaker, text)
   261
   262	        # Configure sampling (matching non-streaming pipeline parameters)
   263	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID (128258)
   264	        # which doesn't exist in the Spark vocab and would cause generation to run to max_tokens
   265	        sampling_params = SamplingParams(
   266	            temperature=temperature,
   267	            top_k=top_k,
   268	            top_p=top_p,
   269	            max_tokens=max_tokens,
   270	            min_tokens=DEFAULT_MIN_TOKENS,
   271	            repetition_penalty=repetition_penalty,
   272	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
   273	            skip_special_tokens=False,  # Keep BiCodec tokens in output
   274	            seed=seed,
   275	        )
   276
   277	        # BiCodec token buffers (separate for semantic and global)
   278	        semantic_buffer = []
   279	        global_buffer = []
   280	        processed_token_count = 0  # Track how many vLLM tokens we've parsed
   281	        total_audio_chunks = 0
   282
   283	        # BiCodec streaming configuration
   284	        EXPECTED_GLOBAL_COUNT = 32  # BiCodec uses 32 fixed global tokens (per paper)
   285	        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8, -120ms TTFB)
   286	        # OPTIMIZATION: Increased from 24 to 48 to reduce O(n) re-decode calls by 2x.
   287	        # At 50 TPS, 48 tokens = ~960ms audio per chunk. Cuts total decode calls roughly in half.
   288	        DECODE_INTERVAL = 48
   289	        CROSSFADE_MS = 50
   290	        # OPTIMIZATION: Windowed decode - only decode last WINDOW_SIZE tokens instead of ALL.
   291	        # BiCodec WaveGenerator has finite receptive field (~64 tokens based on kernel sizes).
   292	        # Window must be >= receptive_field + DECODE_INTERVAL to avoid boundary artifacts.
   293	        WINDOW_SIZE = 128  # 128 tokens covers receptive field + margin for crossfade
   294
   295	        # Generate tokens with vLLM (streaming)
   296	        if request_id is None:
   297	            import uuid
   298	            request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
   299
   300	        # Reuse pre-warmed token parser (singleton, created once in __init__)
   301	        token_parser = self.token_parser
   302
   303	        t_gen_start = time.time()
   304	        batch_wall_ms: List[float] = []
   305	        batch_token_counts: List[int] = []
   306	        decode_calls = 0
   307	        decode_wall_ms = 0.0
   308	        decode_gpu_ms = 0.0
   309	        parse_wall_ms = 0.0
   310
   311	        results_generator = self.model.engine.generate(
   312	            prompt=prompt,
   313	            sampling_params=sampling_params,
   314	            request_id=request_id,
   315	        )
   316
   317	        # Track timing
   318	        t_first_token = None
   319	        t_first_chunk_ready = None
   320	        t_first_chunk_decoded = None
   321	        iteration_count = 0
   322	        last_decode_count = 0
   323
   324	        # Track audio samples EMITTED to the user (not just decoded)
   325	        # BiCodec produces ~320 samples per semantic token at 16kHz
   326	        # CRITICAL: This tracks samples that have been YIELDED, not held in tail
   327	        total_samples_emitted_to_user = 0
   328	        # Keep a tail to enable crossfading between consecutive emitted chunks
   329	        previous_chunk_tail: Optional[bytes] = None
   330
   331	        # Stream tokens with BiCodec extraction
   332	        async for request_output in results_generator:
   333	            iteration_count += 1
   334	            t_iter = time.time()
   335
   336	            # Track first token received
   337	            if t_first_token is None:
   338	                t_first_token = t_iter
   339
   340	            # Extract generated token IDs
   341	            generated_ids = request_output.outputs[0].token_ids
   342
   343	            parse_start = time.time()
   344	            # OPTIMIZATION: Only process NEW tokens (incremental parsing)
   345	            # Old code: tokenizer.decode(ALL) + re.findall(ALL) = O(n²) per stream
   346	            # New code: parse only new tokens = O(1) per token
   347	            new_token_ids = generated_ids[processed_token_count:]
   348	            processed_token_count = len(generated_ids)
   349
   350	            # Parse new tokens and append to buffers (in-place modification)
   351	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
   352	            parse_wall_ms += (time.time() - parse_start) * 1000
   353	            batch_token_counts.append(len(new_token_ids))
   354	            batch_wall_ms.append(0 if t_iter is None else (time.time() - t_iter) * 1000)
   355
   356	            # TRUE STREAMING with BiCodec two-phase generation:
   357	            # Phase 1: Generate 32 global tokens (pre-roll)
   358	            # Phase 2: Stream semantic tokens incrementally
   359
   360	            # Check if we have the 32 global tokens (pre-roll complete)
   361	            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   362	                # Pre-roll complete! Now we can stream semantic tokens
   363	                if t_first_chunk_ready is None and len(semantic_buffer) >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   364	                    t_first_chunk_ready = time.time()
   365
   366	                # Stream semantic tokens incrementally
   367	                # Only decode if we have enough NEW semantic tokens since last decode
   368	                semantic_count = len(semantic_buffer)
   369	                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   370	                    # Check if we should decode (every DECODE_INTERVAL new semantic tokens)
   371	                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
   372	                        last_decode_count = semantic_count
   373
   374	                        # OPTIMIZATION: Windowed decode instead of full re-decode
   375	                        # Only decode the last WINDOW_SIZE tokens (O(1) per chunk instead of O(n))
   376	                        # For first few chunks when total tokens < WINDOW_SIZE, decode all
   377	                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]
   378
   379	                        if len(semantic_buffer) <= WINDOW_SIZE:
   380	                            # Early phase: decode all tokens (window not yet relevant)
   381	                            decode_semantic = semantic_buffer
   382	                        else:
   383	                            # Windowed: only decode last WINDOW_SIZE tokens
   384	                            decode_semantic = semantic_buffer[-WINDOW_SIZE:]
   385
   386	                        t_decode_start = time.time()
   387	                        if emit_timing and torch.cuda.is_available():
   388	                            torch.cuda.synchronize()
   389	                            decode_start_event = torch.cuda.Event(enable_timing=True)
   390	                            decode_end_event = torch.cuda.Event(enable_timing=True)
   391	                            decode_start_event.record()
   392	                        audio_bytes = await self.snac_decoder.decode_single_async(
   393	                            semantic_ids=decode_semantic,
   394	                            global_ids=window_global,
   395	                            trim_warmup=False,
   396	                            use_sliding_window=False,
   397	                        )
   398	                        t_decode_end = time.time()
   399	                        decode_calls += 1
   400	                        decode_wall_ms += (t_decode_end - t_decode_start) * 1000
   401	                        if emit_timing and torch.cuda.is_available():
   402	                            decode_end_event.record()
   403	                            torch.cuda.synchronize()
   404	                            decode_gpu_ms += decode_start_event.elapsed_time(decode_end_event)
   405
   406	                        if audio_bytes:
   407	                            total_samples_decoded = len(audio_bytes) // 2
   408
   409	                            if len(semantic_buffer) <= WINDOW_SIZE:
   410	                                # Early phase: extract new samples from full decode
   411	                                if total_samples_decoded > total_samples_emitted_to_user:
   412	                                    new_bytes_start = total_samples_emitted_to_user * 2
   413	                                    new_audio_bytes = audio_bytes[new_bytes_start:]
   414	                                else:
   415	                                    new_audio_bytes = b""
   416	                            else:
   417	                                # Windowed phase: take only the tail portion of the window decode
   418	                                # The last DECODE_INTERVAL tokens' worth of audio is the new content
   419	                                # Each token produces ~320 samples at 16kHz/50TPS
   420	                                new_samples_approx = DECODE_INTERVAL * 320
   421	                                if total_samples_decoded > new_samples_approx:
   422	                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
   423	                                else:
   424	                                    new_bytes_start = 0
   425	                                new_audio_bytes = audio_bytes[new_bytes_start:]
   426
   427	                            if new_audio_bytes:
   428	                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   429	                                    previous_chunk_tail,
   430	                                    new_audio_bytes,
   431	                                    sample_rate_hz=16000,
   432	                                    crossfade_ms=CROSSFADE_MS,
   433	                                )
   434
   435	                                samples_emitted_in_this_chunk = len(to_emit) // 2
   436	                                total_samples_emitted_to_user += samples_emitted_in_this_chunk
   437
   438	                                if to_emit:
   439	                                    total_audio_chunks += 1
   440	                                    if t_first_chunk_decoded is None:
   441	                                        t_first_chunk_decoded = t_decode_end
   442
   443	                                    if emit_timing:
   444	                                        logger.info(
   445	                                            "tts_stream_decode_batch",
   446	                                            extra={
   447	                                                "speaker": speaker,
   448	                                                "request_id": request_id,
   449	                                                "batch_index": decode_calls,
   450	                                                "decode_wall_ms": int((t_decode_end - t_decode_start) * 1000),
   451	                                                "decode_gpu_ms": int(decode_start_event.elapsed_time(decode_end_event))
   452	                                                if torch.cuda.is_available() else None,
   453	                                                "semantic_count": len(semantic_buffer),
   454	                                            },
   455	                                        )
   456
   457	                                    yield to_emit
   458
   459	        # Final chunk: decode any remaining semantic tokens
   460	        # Use the same 32 global tokens (they get pooled)


Summary: auto


cd /home/ubuntu/veenaModal && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '460,560p'


Output:
Chunk ID: 9d6e1c
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 1326
Output:
   460	        # Use the same 32 global tokens (they get pooled)
   461	        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   462	            audio_bytes = self.snac_decoder.decode_streaming(
   463	                semantic_ids=semantic_buffer,  # All semantic tokens
   464	                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],  # Use first 32 global tokens
   465	                use_sliding_window=False,  # Get all samples
   466	                trim_warmup=False
   467	            )
   468	            if audio_bytes:
   469	                # Only yield NEW samples
   470	                total_samples_decoded = len(audio_bytes) // 2
   471
   472	                if total_samples_decoded > total_samples_emitted_to_user:
   473	                    new_bytes_start = total_samples_emitted_to_user * 2
   474	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   475
   476	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   477	                        previous_chunk_tail,
   478	                        new_audio_bytes,
   479	                        sample_rate_hz=16000,
   480	                        crossfade_ms=CROSSFADE_MS,
   481	                    )
   482
   483	                    samples_emitted_in_final = len(to_emit) // 2
   484	                    total_samples_emitted_to_user += samples_emitted_in_final
   485
   486	                    if to_emit:
   487	                        total_audio_chunks += 1
   488	                        yield to_emit
   489
   490	        # Flush any remaining tail that was held back for crossfade
   491	        if previous_chunk_tail:
   492	            total_audio_chunks += 1
   493	            tail_samples = len(previous_chunk_tail) // 2
   494	            total_samples_emitted_to_user += tail_samples
   495	            yield previous_chunk_tail
   496
   497	        # Log completion summary
   498	        t_end = time.time()
   499	        audio_duration_s = total_samples_emitted_to_user / 16000  # 16kHz sample rate
   500	        ttfb_ms = (t_first_chunk_decoded - t_start) * 1000 if t_first_chunk_decoded else 0
   501	        print(f"🎵 Streaming complete: {audio_duration_s:.2f}s audio, TTFB: {ttfb_ms:.0f}ms, {total_audio_chunks} chunks")
   502
   503	    async def generate_speech_stream_indic_first_chunk(
   504	        self,
   505	        speaker: str,
   506	        text: str,
   507	        temperature: float = DEFAULT_TEMPERATURE,
   508	        top_k: int = DEFAULT_TOP_K,
   509	        top_p: float = DEFAULT_TOP_P,
   510	        max_tokens: int = DEFAULT_MAX_TOKENS,
   511	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   512	        seed: Optional[int] = None,
   513	    ) -> AsyncGenerator[tuple, None]:
   514	        """
   515	        Generate speech for the FIRST chunk of multi-chunk text, capturing global tokens.
   516
   517	        This method is specifically for chunked generation: it yields (audio_bytes, global_ids)
   518	        tuples instead of just audio_bytes. The caller captures global_ids from the first
   519	        yield and passes them to generate_speech_stream_indic_continuation() for subsequent chunks.
   520
   521	        Use Case (chunked streaming with voice consistency):
   522	            globals_captured = None
   523	            async for audio_bytes, global_ids in pipeline.generate_speech_stream_indic_first_chunk(...):
   524	                if globals_captured is None and global_ids:
   525	                    globals_captured = global_ids  # Capture once
   526	                yield audio_bytes
   527	            # Now use globals_captured for subsequent chunks
   528
   529	        Args:
   530	            speaker: Speaker name
   531	            text: First text chunk to synthesize
   532	            temperature: Sampling temperature
   533	            top_k: Top-k sampling
   534	            top_p: Nucleus sampling
   535	            max_tokens: Max tokens to generate
   536	            repetition_penalty: Prevent repetition
   537	            seed: Random seed for reproducibility
   538
   539	        Yields:
   540	            Tuple of (audio_bytes, global_ids)
   541	            - audio_bytes: Raw PCM audio (int16, 16kHz)
   542	            - global_ids: List of 32 global token IDs (populated after first decode, else empty)
   543
   544	        Thread Safety:
   545	            This method is stateless and thread-safe. Each call creates its own
   546	            request-scoped state. No global state is modified or shared.
   547	        """
   548	        import time
   549	        t_start = time.time()
   550
   551	        # Build prompt using Indic prompt builder
   552	        prompt = self.prompt_builder.build_prefix(speaker, text)
   553
   554	        # Configure sampling
   555	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
   556	        sampling_params = SamplingParams(
   557	            temperature=temperature,
   558	            top_k=top_k,
   559	            top_p=top_p,
   560	            max_tokens=max_tokens,


I found the two primary correctness gaps: missing `request_id` propagation in API/WS call sites and indentation/flow issues in runtime timing paths. I’m now reading the remaining streaming helper tail so I can align request IDs + metrics consistently across chunked streaming too.


cd /home/ubuntu/veenaModal && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '560,760p'


Output:
Chunk ID: 0476cf
Wall time: 0.0534 seconds
Process exited with code 0
Original token count: 2688
Output:
   560	            max_tokens=max_tokens,
   561	            min_tokens=DEFAULT_MIN_TOKENS,
   562	            repetition_penalty=repetition_penalty,
   563	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
   564	            skip_special_tokens=False,
   565	            seed=seed,
   566	        )
   567
   568	        # BiCodec token buffers
   569	        semantic_buffer = []
   570	        global_buffer = []
   571	        processed_token_count = 0  # Track how many vLLM tokens we've parsed
   572	        total_audio_chunks = 0
   573
   574	        # BiCodec streaming configuration
   575	        EXPECTED_GLOBAL_COUNT = 32
   576	        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
   577	        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
   578	        CROSSFADE_MS = 50
   579
   580	        # Generate unique request ID
   581	        import uuid
   582	        request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
   583
   584	        # Reuse pre-warmed token parser (singleton, created once in __init__)
   585	        token_parser = self.token_parser
   586
   587	        results_generator = self.model.engine.generate(
   588	            prompt=prompt,
   589	            sampling_params=sampling_params,
   590	            request_id=request_id,
   591	        )
   592
   593	        # Track state
   594	        last_decode_count = 0
   595	        total_samples_emitted_to_user = 0
   596	        previous_chunk_tail: Optional[bytes] = None
   597	        captured_globals: List[int] = []  # Will be populated once we have 32 globals
   598
   599	        # Stream tokens with BiCodec extraction
   600	        async for request_output in results_generator:
   601	            generated_ids = request_output.outputs[0].token_ids
   602
   603	            # OPTIMIZATION: Incremental parsing (O(1) per new token)
   604	            new_token_ids = generated_ids[processed_token_count:]
   605	            processed_token_count = len(generated_ids)
   606	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
   607
   608	            # Capture global tokens once we have all 32
   609	            if not captured_globals and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   610	                captured_globals = global_buffer[:EXPECTED_GLOBAL_COUNT]
   611
   612	            # Process if we have all global tokens
   613	            if len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   614	                semantic_count = len(semantic_buffer)
   615	                if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   616	                    if semantic_count - last_decode_count >= DECODE_INTERVAL:
   617	                        last_decode_count = semantic_count
   618
   619	                        all_semantic = semantic_buffer
   620	                        window_global = global_buffer[:EXPECTED_GLOBAL_COUNT]
   621
   622	                        if self.snac_decoder.enable_batching:
   623	                            audio_bytes = await self.snac_decoder.decode_single_async(
   624	                                semantic_ids=all_semantic,
   625	                                global_ids=window_global,
   626	                                trim_warmup=False,
   627	                                use_sliding_window=False
   628	                            )
   629	                        else:
   630	                            audio_bytes = self.snac_decoder.decode_streaming(
   631	                                semantic_ids=all_semantic,
   632	                                global_ids=window_global,
   633	                                use_sliding_window=False,
   634	                                trim_warmup=False
   635	                            )
   636
   637	                        if audio_bytes:
   638	                            total_samples_decoded = len(audio_bytes) // 2
   639
   640	                            if total_samples_decoded > total_samples_emitted_to_user:
   641	                                new_bytes_start = total_samples_emitted_to_user * 2
   642	                                new_audio_bytes = audio_bytes[new_bytes_start:]
   643
   644	                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   645	                                    previous_chunk_tail,
   646	                                    new_audio_bytes,
   647	                                    sample_rate_hz=16000,
   648	                                    crossfade_ms=CROSSFADE_MS,
   649	                                )
   650
   651	                                samples_emitted_in_this_chunk = len(to_emit) // 2
   652	                                total_samples_emitted_to_user += samples_emitted_in_this_chunk
   653
   654	                                if to_emit:
   655	                                    total_audio_chunks += 1
   656	                                    # Yield tuple: (audio_bytes, global_ids)
   657	                                    yield (to_emit, captured_globals)
   658
   659	        # Final chunk processing
   660	        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= EXPECTED_GLOBAL_COUNT:
   661	            audio_bytes = self.snac_decoder.decode_streaming(
   662	                semantic_ids=semantic_buffer,
   663	                global_ids=global_buffer[:EXPECTED_GLOBAL_COUNT],
   664	                use_sliding_window=False,
   665	                trim_warmup=False
   666	            )
   667	            if audio_bytes:
   668	                total_samples_decoded = len(audio_bytes) // 2
   669
   670	                if total_samples_decoded > total_samples_emitted_to_user:
   671	                    new_bytes_start = total_samples_emitted_to_user * 2
   672	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   673
   674	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   675	                        previous_chunk_tail,
   676	                        new_audio_bytes,
   677	                        sample_rate_hz=16000,
   678	                        crossfade_ms=CROSSFADE_MS,
   679	                    )
   680
   681	                    samples_emitted_in_final = len(to_emit) // 2
   682	                    total_samples_emitted_to_user += samples_emitted_in_final
   683
   684	                    if to_emit:
   685	                        total_audio_chunks += 1
   686	                        yield (to_emit, captured_globals)
   687
   688	        # Flush remaining tail
   689	        if previous_chunk_tail:
   690	            total_audio_chunks += 1
   691	            tail_samples = len(previous_chunk_tail) // 2
   692	            total_samples_emitted_to_user += tail_samples
   693	            yield (previous_chunk_tail, captured_globals)
   694
   695	        t_end = time.time()
   696	        audio_duration_s = total_samples_emitted_to_user / 16000
   697	        print(f"🎵 First chunk complete: {audio_duration_s:.2f}s audio, {total_audio_chunks} chunks, captured {len(captured_globals)} globals")
   698
   699	    async def generate_speech_stream_indic_continuation(
   700	        self,
   701	        speaker: str,
   702	        text: str,
   703	        global_ids: List[int],
   704	        temperature: float = DEFAULT_TEMPERATURE,
   705	        top_k: int = DEFAULT_TOP_K,
   706	        top_p: float = DEFAULT_TOP_P,
   707	        max_tokens: int = DEFAULT_MAX_TOKENS,
   708	        repetition_penalty: float = DEFAULT_REPETITION_PENALTY,
   709	        seed: Optional[int] = None,
   710	    ) -> AsyncGenerator[bytes, None]:
   711	        """
   712	        Generate speech for CONTINUATION chunks using pre-captured global tokens.
   713
   714	        This is the key method for voice consistency in chunked generation:
   715	        - Uses global_ids from the first chunk to maintain identical voice
   716	        - Model generates only semantic tokens (skips global token generation)
   717	        - Ensures no voice drift across chunks
   718
   719	        CRITICAL for production:
   720	        - global_ids is request-scoped, passed explicitly (no shared state)
   721	        - Thread-safe: each request has its own state
   722	        - No global caching that could cause cross-request contamination
   723
   724	        Args:
   725	            speaker: Speaker name (must match first chunk)
   726	            text: Continuation text chunk to synthesize
   727	            global_ids: 32 global token IDs captured from first chunk
   728	            temperature: Sampling temperature
   729	            top_k: Top-k sampling
   730	            top_p: Nucleus sampling
   731	            max_tokens: Max tokens to generate
   732	            repetition_penalty: Prevent repetition
   733	            seed: Random seed (use same as first chunk for consistency)
   734
   735	        Yields:
   736	            Audio bytes (int16 PCM, 16kHz mono)
   737
   738	        Raises:
   739	            ValueError: If global_ids doesn't contain exactly 32 tokens
   740
   741	        Thread Safety:
   742	            Fully thread-safe. All state is request-scoped and passed explicitly.
   743	        """
   744	        import time
   745	        t_start = time.time()
   746
   747	        # Validate global tokens
   748	        EXPECTED_GLOBAL_COUNT = 32
   749	        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
   750	            raise ValueError(
   751	                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
   752	                f"global_ids must be captured from first chunk generation."
   753	            )
   754
   755	        # Build prompt WITH pre-filled global tokens
   756	        # This tells the model to skip global generation and go straight to semantic tokens
   757	        prompt = self.prompt_builder.build_prefix_with_globals(speaker, text, global_ids)
   758
   759	        # Configure sampling
   760	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID


cd /home/ubuntu/veenaModal && nl -ba veena3modal/core/streaming_pipeline.py | sed -n '760,892p'


Output:
Chunk ID: 7d9f83
Wall time: 0.0528 seconds
Process exited with code 0
Original token count: 1802
Output:
   760	        # OPTIMIZATION: Use Spark TTS stop token, not legacy SNAC CODE_END_TOKEN_ID
   761	        sampling_params = SamplingParams(
   762	            temperature=temperature,
   763	            top_k=top_k,
   764	            top_p=top_p,
   765	            max_tokens=max_tokens,
   766	            min_tokens=DEFAULT_MIN_TOKENS,
   767	            repetition_penalty=repetition_penalty,
   768	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>" - matches non-streaming pipeline
   769	            skip_special_tokens=False,
   770	            seed=seed,
   771	        )
   772
   773	        # BiCodec token buffers
   774	        semantic_buffer = []
   775	        global_buffer_unused = []  # Not used in continuation, but needed for parser API
   776	        processed_token_count = 0  # Track how many vLLM tokens we've parsed
   777	        total_audio_chunks = 0
   778
   779	        # BiCodec streaming configuration
   780	        MIN_SEMANTIC_FOR_FIRST_CHUNK = 10  # OPTIMIZATION: Lowered from 16 (decoder min is 8)
   781	        DECODE_INTERVAL = 48  # OPTIMIZATION: Increased from 24 to reduce O(n) re-decode calls
   782	        CROSSFADE_MS = 50
   783
   784	        # Generate unique request ID
   785	        import uuid
   786	        request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
   787
   788	        # Reuse pre-warmed token parser (singleton, created once in __init__)
   789	        token_parser = self.token_parser
   790
   791	        results_generator = self.model.engine.generate(
   792	            prompt=prompt,
   793	            sampling_params=sampling_params,
   794	            request_id=request_id,
   795	        )
   796
   797	        # Track state
   798	        last_decode_count = 0
   799	        total_samples_emitted_to_user = 0
   800	        previous_chunk_tail: Optional[bytes] = None
   801
   802	        # Stream tokens - model will generate ONLY semantic tokens (globals are pre-filled)
   803	        async for request_output in results_generator:
   804	            generated_ids = request_output.outputs[0].token_ids
   805
   806	            # OPTIMIZATION: Incremental parsing (O(1) per new token)
   807	            # Note: In continuation, model only generates semantic tokens (globals pre-filled in prompt)
   808	            new_token_ids = generated_ids[processed_token_count:]
   809	            processed_token_count = len(generated_ids)
   810	            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer_unused)
   811
   812	            semantic_count = len(semantic_buffer)
   813	            if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
   814	                if semantic_count - last_decode_count >= DECODE_INTERVAL:
   815	                    last_decode_count = semantic_count
   816
   817	                    # Use pre-captured global tokens for decoding
   818	                    if self.snac_decoder.enable_batching:
   819	                        audio_bytes = await self.snac_decoder.decode_single_async(
   820	                            semantic_ids=semantic_buffer,
   821	                            global_ids=global_ids,  # Use captured globals
   822	                            trim_warmup=False,
   823	                            use_sliding_window=False
   824	                        )
   825	                    else:
   826	                        audio_bytes = self.snac_decoder.decode_streaming(
   827	                            semantic_ids=semantic_buffer,
   828	                            global_ids=global_ids,  # Use captured globals
   829	                            use_sliding_window=False,
   830	                            trim_warmup=False
   831	                        )
   832
   833	                    if audio_bytes:
   834	                        total_samples_decoded = len(audio_bytes) // 2
   835
   836	                        if total_samples_decoded > total_samples_emitted_to_user:
   837	                            new_bytes_start = total_samples_emitted_to_user * 2
   838	                            new_audio_bytes = audio_bytes[new_bytes_start:]
   839
   840	                            to_emit, previous_chunk_tail = crossfade_bytes_int16(
   841	                                previous_chunk_tail,
   842	                                new_audio_bytes,
   843	                                sample_rate_hz=16000,
   844	                                crossfade_ms=CROSSFADE_MS,
   845	                            )
   846
   847	                            samples_emitted_in_this_chunk = len(to_emit) // 2
   848	                            total_samples_emitted_to_user += samples_emitted_in_this_chunk
   849
   850	                            if to_emit:
   851	                                total_audio_chunks += 1
   852	                                yield to_emit
   853
   854	        # Final chunk processing
   855	        if len(semantic_buffer) > last_decode_count:
   856	            audio_bytes = self.snac_decoder.decode_streaming(
   857	                semantic_ids=semantic_buffer,
   858	                global_ids=global_ids,  # Use captured globals
   859	                use_sliding_window=False,
   860	                trim_warmup=False
   861	            )
   862	            if audio_bytes:
   863	                total_samples_decoded = len(audio_bytes) // 2
   864
   865	                if total_samples_decoded > total_samples_emitted_to_user:
   866	                    new_bytes_start = total_samples_emitted_to_user * 2
   867	                    new_audio_bytes = audio_bytes[new_bytes_start:]
   868
   869	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   870	                        previous_chunk_tail,
   871	                        new_audio_bytes,
   872	                        sample_rate_hz=16000,
   873	                        crossfade_ms=CROSSFADE_MS,
   874	                    )
   875
   876	                    samples_emitted_in_final = len(to_emit) // 2
   877	                    total_samples_emitted_to_user += samples_emitted_in_final
   878
   879	                    if to_emit:
   880	                        total_audio_chunks += 1
   881	                        yield to_emit
   882
   883	        # Flush remaining tail
   884	        if previous_chunk_tail:
   885	            total_audio_chunks += 1
   886	            tail_samples = len(previous_chunk_tail) // 2
   887	            total_samples_emitted_to_user += tail_samples
   888	            yield previous_chunk_tail
   889
   890	        t_end = time.time()
   891	        audio_duration_s = total_samples_emitted_to_user / 16000
   892	        print(f"🎵 Continuation chunk complete: {audio_duration_s:.2f}s audio, {total_audio_chunks} chunks (using cached globals)")


cd /home/ubuntu/veenaModal && rg -n "benchmark|stress|gpu|concurrency|load" scripts veena3modal | head -n 80


Output:
Chunk ID: 632f3d
Wall time: 0.0502 seconds
Process exited with code 0
Original token count: 1771
Output:
veena3modal/local_server.py:6:2. Downloads model weights from HuggingFace if not present
veena3modal/local_server.py:11:    # Basic (auto-downloads model, uses defaults)
veena3modal/local_server.py:14:    # Custom model path (skip download)
veena3modal/local_server.py:18:    python -m veena3modal.local_server --gpu-memory 0.5 --port 8080
veena3modal/local_server.py:71:# 2. HuggingFace download (private repo: BayAreaBoys/spark_tts_4speaker)
veena3modal/local_server.py:77:def download_model(
veena3modal/local_server.py:83:    Download Spark TTS model from HuggingFace if not already present.
veena3modal/local_server.py:88:    3. Provide instructions for Modal volume download if both fail
veena3modal/local_server.py:99:        from huggingface_hub import snapshot_download
veena3modal/local_server.py:103:            "Or download model manually and pass --model-path"
veena3modal/local_server.py:110:    logger.info(f"Attempting download from {hf_repo}...")
veena3modal/local_server.py:112:        downloaded_path = snapshot_download(
veena3modal/local_server.py:118:        logger.info(f"Model downloaded to {downloaded_path}")
veena3modal/local_server.py:119:        return downloaded_path
veena3modal/local_server.py:121:        logger.warning(f"Private repo download failed: {e}")
veena3modal/local_server.py:127:            downloaded_path = snapshot_download(
veena3modal/local_server.py:132:            logger.info(f"Base model downloaded to {downloaded_path}")
veena3modal/local_server.py:135:                "For full speaker support, download your fine-tuned model."
veena3modal/local_server.py:137:            return downloaded_path
veena3modal/local_server.py:139:            logger.warning(f"Public model download also failed: {e2}")
veena3modal/local_server.py:141:    # All downloads failed - provide manual instructions
veena3modal/local_server.py:143:        "\nModel download failed. Options:\n"
veena3modal/local_server.py:144:        "  1. Download from Modal volume:\n"
veena3modal/local_server.py:146:        "  2. Download from HuggingFace (if you have access):\n"
veena3modal/local_server.py:147:        f"     huggingface-cli download {PRIVATE_HF_REPO} --local-dir {local_dir}\n\n"
veena3modal/local_server.py:149:        f"     huggingface-cli download {PUBLIC_FALLBACK_REPO} --local-dir {local_dir}\n\n"
veena3modal/local_server.py:162:        LLM/          <- Language model (vLLM loads this)
veena3modal/local_server.py:195:    gpu_memory_utilization: float = 0.85,
veena3modal/local_server.py:219:    logger.info(f"  GPU Memory:  {gpu_memory_utilization:.0%} (total)")
veena3modal/local_server.py:230:        gpu_memory_utilization=gpu_memory_utilization,
veena3modal/local_server.py:239:    logger.info(f"  Super-resolution: {'loaded' if runtime.sr_service and runtime.sr_service.is_loaded else 'disabled'}")
veena3modal/local_server.py:261:  python -m veena3modal.local_server                          # Auto-download model, serve on :8000
veena3modal/local_server.py:264:  python -m veena3modal.local_server --gpu-memory 0.5         # Limit GPU memory
veena3modal/local_server.py:265:  python -m veena3modal.local_server --workers 2              # Multiple workers (careful: each loads model)
veena3modal/local_server.py:281:        help=f"HuggingFace repo to download from (default: {PRIVATE_HF_REPO})",
veena3modal/local_server.py:290:        "--skip-download",
veena3modal/local_server.py:292:        help="Don't auto-download model (fail if not found)",
veena3modal/local_server.py:296:    gpu_group = parser.add_argument_group("GPU")
veena3modal/local_server.py:297:    gpu_group.add_argument(
veena3modal/local_server.py:304:    gpu_group.add_argument(
veena3modal/local_server.py:305:        "--gpu-memory",
veena3modal/local_server.py:310:    gpu_group.add_argument(
veena3modal/local_server.py:350:        help="Number of uvicorn workers (default: 1, each loads full model)",
veena3modal/local_server.py:353:        "--reload",
veena3modal/local_server.py:356:        help="Enable auto-reload for development (incompatible with --workers > 1)",
veena3modal/local_server.py:387:    2. Download model if needed
veena3modal/local_server.py:416:        if args.skip_download:
veena3modal/local_server.py:417:            logger.error(f"Model not found at {model_path} and --skip-download is set")
veena3modal/local_server.py:419:        model_path = download_model(
veena3modal/local_server.py:425:    # Initialize TTS runtime (this is the expensive step - loads model onto GPU)
veena3modal/local_server.py:429:        gpu_memory_utilization=args.gpu_memory,
veena3modal/local_server.py:438:    # With --reload, uvicorn reimports, but the runtime singleton persists in memory.
veena3modal/local_server.py:457:        reload=args.reload,
veena3modal/local_server.py:459:        # Performance: keep-alive for connection reuse under load
veena3modal/app.py:7:- Per-container concurrency (vLLM batching)
veena3modal/app.py:118:# Models should be uploaded to this volume before deployment
veena3modal/app.py:143:    # - Memory-bound workload, so faster GPUs have diminishing returns
veena3modal/app.py:144:    gpu="L40S",
veena3modal/app.py:171:    startup_timeout=600,     # 10 min for model loading (reduced from 20 min)
veena3modal/app.py:186:    Uses @modal.enter for model loading (once per container).
veena3modal/app.py:191:    def load_model(self):
veena3modal/app.py:204:        logger.info("🚀 Starting TTS model loading...")
veena3modal/app.py:230:                gpu_memory_utilization=0.85,
veena3modal/app.py:234:            logger.info(f"✅ TTS model loaded: {self.runtime.model_version}")
veena3modal/app.py:237:            logger.error(f"❌ Model loading failed: {e}")
veena3modal/app.py:255:    gpu="L40S",  # Best cost/performance for TTS
veena3modal/app.py:264:    startup_timeout=600,     # 10 min for model load
veena3modal/app.py:271:    Note: Model will be loaded on first request (higher TTFB).
veena3modal/app.py:297:    gpu="L40S",
veena3modal/services/tts_runtime.py:4:In Modal, each container should load the model once and then handle many requests.
veena3modal/services/tts_runtime.py:52:        This class is designed for async concurrency within a single container.
veena3modal/services/tts_runtime.py:63:    model_version: str = "not_loaded"
veena3modal/services/tts_runtime.py:64:    is_loaded: bool = False
veena3modal/services/tts_runtime.py:65:    load_time_ms: float = 0.0
veena3modal/services/tts_runtime.py:89:    return _runtime is not None and _runtime.is_loaded
veena3modal/services/tts_runtime.py:98:    gpu_memory_utilization: float = 0.25,
veena3modal/services/tts_runtime.py:113:        gpu_memory_utilization: vLLM GPU memory fraction (default: 0.25)
veena3modal/services/tts_runtime.py:146:        from veena3modal.core.model_loader import SparkTTSModel
veena3modal/services/tts_runtime.py:154:            # Each engine gets gpu_memory_utilization / num_engines to share GPU fairly.
veena3modal/services/tts_runtime.py:157:            per_engine_mem = gpu_memory_utilization / num_engines


cd /home/ubuntu/veenaModal && sed -n '1,260p' veena3modal/local_server.py


Output:
Chunk ID: 994954
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2380
Output:
"""
Local entry point for Veena3 TTS - runs the same FastAPI app without Modal.

Replicates what Modal's @modal.enter + @modal.asgi_app does:
1. Sets up PYTHONPATH for vendored deps (sparktts, AP-BWE)
2. Downloads model weights from HuggingFace if not present
3. Initializes TTS runtime (vLLM engine, BiCodec decoder, pipelines)
4. Serves the FastAPI app via uvicorn

Usage:
    # Basic (auto-downloads model, uses defaults)
    python -m veena3modal.local_server

    # Custom model path (skip download)
    python -m veena3modal.local_server --model-path /path/to/spark_tts_4speaker

    # Custom GPU memory + port
    python -m veena3modal.local_server --gpu-memory 0.5 --port 8080

    # With super-resolution (48kHz output)
    python -m veena3modal.local_server --enable-sr --sr-path /path/to/ap_bwe/16kto48k

    # CPU-only (no GPU, for testing only - very slow)
    python -m veena3modal.local_server --device cpu

Environment Variables (override defaults):
    MODEL_PATH              - Path to Spark TTS model directory
    HF_TOKEN                - HuggingFace token for private models
    AP_BWE_CHECKPOINT_DIR   - Path to super-resolution checkpoints
    AUTH_BYPASS_MODE        - "true" to disable auth (default for local)
    GPU_MEMORY_UTILIZATION  - vLLM GPU memory fraction (default: 0.25)
"""

from __future__ import annotations

import argparse
import logging
import os
import sys
import time

# === Path Setup (MUST happen before any veena3modal imports) ===
# Vendored deps: sparktts and AP-BWE live in external/ relative to repo root
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
EXTERNAL_DIR = os.path.join(REPO_ROOT, "external")

# sparktts package path: external/sparktts (contains sparktts/ subpackage)
SPARKTTS_PATH = os.path.join(EXTERNAL_DIR, "sparktts")
# AP-BWE path: external/AP-BWE
AP_BWE_PATH = os.path.join(EXTERNAL_DIR, "AP-BWE")

for path in [SPARKTTS_PATH, AP_BWE_PATH, REPO_ROOT]:
    if path not in sys.path:
        sys.path.insert(0, path)

# Also set PYTHONPATH for subprocesses (e.g., ffmpeg wrappers)
existing_pythonpath = os.environ.get("PYTHONPATH", "")
new_paths = f"{SPARKTTS_PATH}:{AP_BWE_PATH}:{REPO_ROOT}"
if existing_pythonpath:
    os.environ["PYTHONPATH"] = f"{new_paths}:{existing_pythonpath}"
else:
    os.environ["PYTHONPATH"] = new_paths

logger = logging.getLogger("veena3.local")

# Default local model directory (relative to repo)
DEFAULT_LOCAL_MODEL_DIR = os.path.join(REPO_ROOT, "models", "spark_tts_4speaker")

# Model sources (tried in order):
# 1. Local path (--model-path or MODEL_PATH env)
# 2. HuggingFace download (private repo: BayAreaBoys/spark_tts_4speaker)
# 3. Base Spark TTS model (public: SparkAudio/Spark-TTS-0.5B)
PRIVATE_HF_REPO = "BayAreaBoys/spark_tts_4speaker"
PUBLIC_FALLBACK_REPO = "SparkAudio/Spark-TTS-0.5B"

def download_model(
    hf_repo: str = PRIVATE_HF_REPO,
    local_dir: str = DEFAULT_LOCAL_MODEL_DIR,
    hf_token: str | None = None,
) -> str:
    """
    Download Spark TTS model from HuggingFace if not already present.

    Strategy:
    1. Try private repo (BayAreaBoys/spark_tts_4speaker) with HF token
    2. Fall back to public base model (SparkAudio/Spark-TTS-0.5B)
    3. Provide instructions for Modal volume download if both fail

    Returns the resolved local path.
    """
    # Check for any valid marker file (config.json at root, config.yaml, or LLM/config.json)
    marker_files = ["config.json", "config.yaml", "LLM/config.json"]
    if any(os.path.exists(os.path.join(local_dir, m)) for m in marker_files):
        logger.info(f"Model already present at {local_dir}")
        return local_dir

    try:
        from huggingface_hub import snapshot_download
    except ImportError:
        logger.error(
            "huggingface_hub not installed. Install with: pip install huggingface-hub\n"
            "Or download model manually and pass --model-path"
        )
        sys.exit(1)

    os.makedirs(local_dir, exist_ok=True)

    # Try private repo first
    logger.info(f"Attempting download from {hf_repo}...")
    try:
        downloaded_path = snapshot_download(
            repo_id=hf_repo,
            local_dir=local_dir,
            token=hf_token,
            ignore_patterns=["*.bin", "training_args.bin", "optimizer.pt"],
        )
        logger.info(f"Model downloaded to {downloaded_path}")
        return downloaded_path
    except Exception as e:
        logger.warning(f"Private repo download failed: {e}")

    # Try public fallback
    if hf_repo != PUBLIC_FALLBACK_REPO:
        logger.info(f"Trying public base model: {PUBLIC_FALLBACK_REPO}...")
        try:
            downloaded_path = snapshot_download(
                repo_id=PUBLIC_FALLBACK_REPO,
                local_dir=local_dir,
                token=hf_token,
            )
            logger.info(f"Base model downloaded to {downloaded_path}")
            logger.warning(
                "Using public base SparkTTS model (no custom speakers). "
                "For full speaker support, download your fine-tuned model."
            )
            return downloaded_path
        except Exception as e2:
            logger.warning(f"Public model download also failed: {e2}")

    # All downloads failed - provide manual instructions
    logger.error(
        "\nModel download failed. Options:\n"
        "  1. Download from Modal volume:\n"
        "     modal volume get veena3-models spark_tts_4speaker models/spark_tts_4speaker\n\n"
        "  2. Download from HuggingFace (if you have access):\n"
        f"     huggingface-cli download {PRIVATE_HF_REPO} --local-dir {local_dir}\n\n"
        "  3. Use base SparkTTS model:\n"
        f"     huggingface-cli download {PUBLIC_FALLBACK_REPO} --local-dir {local_dir}\n\n"
        "  4. Specify a custom path:\n"
        "     python -m veena3modal.local_server --model-path /path/to/model"
    )
    sys.exit(1)

def resolve_model_paths(model_path: str) -> tuple[str, str]:
    """
    Resolve LLM and BiCodec paths from a Spark TTS model directory.

    Spark TTS model structure (from SparkAudio/Spark-TTS-0.5B):
      model_path/
        LLM/          <- Language model (vLLM loads this)
        BiCodec/      <- Audio tokenizer/decoder
        config.yaml   <- Top-level config

    Fine-tuned models may have LLM files at root (flat structure).
    This function auto-detects both layouts.

    Returns:
        (llm_path, bicodec_path) resolved paths
    """
    # Check for subdirectory layout (SparkAudio/Spark-TTS-0.5B style)
    llm_subdir = os.path.join(model_path, "LLM")
    bicodec_subdir = os.path.join(model_path, "BiCodec")

    if os.path.isdir(llm_subdir) and os.path.exists(os.path.join(llm_subdir, "config.json")):
        llm_path = llm_subdir
    else:
        # Flat layout: LLM files at root
        llm_path = model_path

    # BiCodec path is always the root (it contains BiCodec/ subdir)
    bicodec_path = model_path

    logger.info(f"Resolved model paths:")
    logger.info(f"  LLM (vLLM):    {llm_path}")
    logger.info(f"  BiCodec:       {bicodec_path}")

    return llm_path, bicodec_path

def initialize_local_runtime(
    model_path: str,
    device: str = "cuda",
    gpu_memory_utilization: float = 0.85,
    enable_sr: bool = False,
    sr_checkpoint_dir: str | None = None,
    hf_token: str | None = None,
    num_engines: int = 1,
):
    """
    Initialize TTS runtime for local serving.

    This replaces Modal's @modal.enter lifecycle hook.
    Loads: vLLM engine, BiCodec decoder, prompt builder, pipelines.
    """
    from veena3modal.services.tts_runtime import initialize_runtime

    # Resolve LLM vs BiCodec paths from model directory
    llm_path, bicodec_path = resolve_model_paths(model_path)

    start = time.time()
    logger.info("=" * 60)
    logger.info("Initializing Veena3 TTS Runtime (local)")
    logger.info(f"  Model:       {model_path}")
    logger.info(f"  LLM path:    {llm_path}")
    logger.info(f"  BiCodec:     {bicodec_path}")
    logger.info(f"  Device:      {device}")
    logger.info(f"  GPU Memory:  {gpu_memory_utilization:.0%} (total)")
    logger.info(f"  Engines:     {num_engines}")
    logger.info(f"  SR:          {'enabled' if enable_sr else 'disabled'}")
    logger.info("=" * 60)

    runtime = initialize_runtime(
        model_path=llm_path,
        bicodec_path=bicodec_path,
        sr_checkpoint_dir=sr_checkpoint_dir if enable_sr else None,
        device=device,
        hf_token=hf_token,
        gpu_memory_utilization=gpu_memory_utilization,
        enable_sr=enable_sr,
        num_engines=num_engines,
    )

    elapsed = time.time() - start
    logger.info(f"Runtime initialized in {elapsed:.1f}s")
    logger.info(f"  Model version: {runtime.model_version}")
    logger.info(f"  Streaming: {'available' if runtime.streaming_pipeline else 'unavailable'}")
    logger.info(f"  Super-resolution: {'loaded' if runtime.sr_service and runtime.sr_service.is_loaded else 'disabled'}")

    return runtime

def create_local_app():
    """
    Create the FastAPI app for local serving.

    Same app factory as Modal deployment, but called from local context.
    """
    from veena3modal.api.fastapi_app import create_app
    return create_app()

def parse_args():
    """Parse CLI arguments for local server configuration."""
    parser = argparse.ArgumentParser(
        description="Veena3 TTS Local Server - GPU-accelerated text-to-speech",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:


Output:
Chunk ID: 982008
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 254
Output:
total 84
drwxrwxr-x 11 ubuntu ubuntu  4096 Feb 16 23:49 .
drwxr-x--- 35 ubuntu ubuntu  4096 Feb 16 21:57 ..
drwxrwxr-x  3 ubuntu ubuntu  4096 Feb 13 23:23 .cursor
-rw-rw-r--  1 ubuntu ubuntu    99 Feb 13 23:23 .cursorignore
-rw-rw-r--  1 ubuntu ubuntu 12479 Feb 13 23:23 .cursorrules
-rw-rw-r--  1 ubuntu ubuntu    46 Feb 13 23:32 .env
-rw-rw-r--  1 ubuntu ubuntu   777 Feb 13 23:23 .env.example
drwxrwxr-x  8 ubuntu ubuntu  4096 Feb 13 23:25 .git
drwxrwxr-x  3 ubuntu ubuntu  4096 Feb 13 23:23 .github
-rw-rw-r--  1 ubuntu ubuntu  1393 Feb 13 23:23 .gitignore
-rw-rw-r--  1 ubuntu ubuntu  2096 Feb 13 23:23 README.md
drwxrwxr-x  4 ubuntu ubuntu  4096 Feb 13 23:23 external
drwxrwxr-x  3 ubuntu ubuntu  4096 Feb 13 23:23 modal_docs
drwxrwxr-x  3 ubuntu ubuntu  4096 Feb 13 23:58 models
-rw-rw-r--  1 ubuntu ubuntu   458 Feb 13 23:23 requirements.txt
drwxrwxr-x  3 ubuntu ubuntu  4096 Feb 14 06:39 scripts
drwxrwxr-x 10 ubuntu ubuntu  4096 Feb 14 00:09 veena3modal
drwxrwxr-x  6 ubuntu ubuntu  4096 Feb 13 23:55 venv


cd /home/ubuntu/veenaModal && ls -la scripts && rg -n "benchmark|stress|concur|timing" scripts


Output:
Chunk ID: b18479
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 2614
Output:
total 440
drwxrwxr-x  3 ubuntu ubuntu  4096 Feb 14 06:39 .
drwxrwxr-x 11 ubuntu ubuntu  4096 Feb 16 23:49 ..
-rw-rw-r--  1 ubuntu ubuntu 13547 Feb 13 23:23 analyze_missing_words.py
-rw-rw-r--  1 ubuntu ubuntu 12017 Feb 13 23:23 asr_validation_openai.py
-rw-rw-r--  1 ubuntu ubuntu  9690 Feb 13 23:23 asr_with_openai_final.py
-rw-rw-r--  1 ubuntu ubuntu 18597 Feb 13 23:23 comprehensive_validation.py
-rw-rw-r--  1 ubuntu ubuntu 18816 Feb 13 23:23 create_visual_map.py
-rw-rw-r--  1 ubuntu ubuntu 20544 Feb 13 23:23 detailed_word_analysis.py
-rw-rw-r--  1 ubuntu ubuntu 11308 Feb 13 23:23 elevenlabs_asr_analysis.py
-rw-rw-r--  1 ubuntu ubuntu 14299 Feb 13 23:23 elevenlabs_verify_fix.py
-rwxrwxr-x  1 ubuntu ubuntu 15162 Feb 13 23:23 final_asr_verification.py
-rwxrwxr-x  1 ubuntu ubuntu  2654 Feb 13 23:23 final_chunking_test.sh
-rwxrwxr-x  1 ubuntu ubuntu  5619 Feb 13 23:23 measure_client_ttfb.py
-rwxrwxr-x  1 ubuntu ubuntu  8576 Feb 13 23:23 measure_ttfb_detailed.py
-rw-rw-r--  1 ubuntu ubuntu  2875 Feb 13 23:23 measure_ttfb_detailed_v2.py
-rw-rw-r--  1 ubuntu ubuntu 14380 Feb 13 23:23 pattern_analysis.py
-rw-rw-r--  1 ubuntu ubuntu 26820 Feb 16 23:29 profile_pipeline.py
-rw-rw-r--  1 ubuntu ubuntu  3638 Feb 13 23:23 quick_asr_check.py
-rwxrwxr-x  1 ubuntu ubuntu  7872 Feb 13 23:57 setup_local.sh
-rw-rw-r--  1 ubuntu ubuntu 23205 Feb 14 00:19 stress_test_local.py
-rw-rw-r--  1 ubuntu ubuntu 17729 Feb 16 23:29 stress_test_production.py
-rw-rw-r--  1 ubuntu ubuntu  9404 Feb 13 23:23 test_all_features.py
-rwxrwxr-x  1 ubuntu ubuntu  9752 Feb 13 23:23 test_chunking_no_emotions.py
-rw-rw-r--  1 ubuntu ubuntu  9486 Feb 13 23:23 test_correct_format.py
-rw-rw-r--  1 ubuntu ubuntu  8167 Feb 13 23:23 test_fix.py
-rw-rw-r--  1 ubuntu ubuntu  9081 Feb 13 23:23 test_original_bug.py
-rwxrwxr-x  1 ubuntu ubuntu  8673 Feb 13 23:23 test_true_streaming.py
-rw-rw-r--  1 ubuntu ubuntu 11966 Feb 13 23:23 validate_chunking_asr.py
-rwxrwxr-x  1 ubuntu ubuntu 15493 Feb 13 23:23 validate_indic_normalization.py
-rw-rw-r--  1 ubuntu ubuntu 10236 Feb 13 23:23 validate_modal_endpoint.py
-rwxrwxr-x  1 ubuntu ubuntu 14508 Feb 13 23:23 validate_text_normalization.py
-rwxrwxr-x  1 ubuntu ubuntu  5986 Feb 13 23:23 validate_true_streaming.py
drwxrwxr-x  2 ubuntu ubuntu  4096 Feb 13 23:23 validation
-rw-rw-r--  1 ubuntu ubuntu 17439 Feb 13 23:23 wer_test_with_emotions.py
scripts/measure_ttfb_detailed_v2.py:3:Measure TTFB with detailed per-chunk timing
scripts/measure_ttfb_detailed.py:3:Measure TTFB (Time To First Byte) with detailed timing breakdown.
scripts/test_true_streaming.py:26:    """Test TRUE streaming with detailed timing."""
scripts/test_true_streaming.py:54:    # Track chunk timing
scripts/test_true_streaming.py:243:    print("\nCheck server logs for detailed timing breakdown:")
scripts/measure_client_ttfb.py:42:    # Start timing
scripts/comprehensive_validation.py:27:from concurrent.futures import ThreadPoolExecutor, as_completed
scripts/comprehensive_validation.py:393:def test_concurrency():
scripts/comprehensive_validation.py:397:    test_text = "Hello! This is a concurrent test."
scripts/comprehensive_validation.py:398:    concurrency_levels = [1, 3, 5]
scripts/comprehensive_validation.py:402:    for n_concurrent in concurrency_levels:
scripts/comprehensive_validation.py:403:        log_info(f"\nTesting with {n_concurrent} concurrent requests...")
scripts/comprehensive_validation.py:416:        with ThreadPoolExecutor(max_workers=n_concurrent) as executor:
scripts/comprehensive_validation.py:417:            futures = [executor.submit(make_request, i) for i in range(n_concurrent)]
scripts/comprehensive_validation.py:428:        log_info(f"  Success: {success_count}/{n_concurrent}")
scripts/comprehensive_validation.py:432:        results[n_concurrent] = {
scripts/comprehensive_validation.py:433:            "success_rate": success_count / n_concurrent,
scripts/comprehensive_validation.py:442:        log_success("All concurrent requests succeeded")
scripts/comprehensive_validation.py:445:        log_warning("Some concurrent requests failed")
scripts/comprehensive_validation.py:501:        ("Concurrency", test_concurrency),
scripts/stress_test_production.py:12:Runs the same concurrency ramp as the baseline test for direct comparison.
scripts/stress_test_production.py:88:# === GPU Monitor (reused from stress_test_local.py) ===
scripts/stress_test_production.py:227:async def run_level(base_url, num_requests, concurrency, stream, gpu_monitor=None):
scripts/stress_test_production.py:228:    sem = asyncio.Semaphore(concurrency)
scripts/stress_test_production.py:243:    limits = httpx.Limits(max_connections=concurrency + 10, max_keepalive_connections=concurrency)
scripts/stress_test_production.py:265:    label = f"{concurrency}c-production-{mode}"
scripts/stress_test_production.py:269:    print(f"  {num_requests} requests, {concurrency} concurrent, {mode}")
scripts/stress_test_production.py:306:        "level": label, "concurrency": concurrency, "requests": num_requests,
scripts/stress_test_production.py:350:            print(f"\n>>> {conc} concurrent, {n} total, real sentences, stream={stream}")
scripts/stress_test_production.py:406:    parser.add_argument("--output", default="stress_test_production.json")
scripts/profile_pipeline.py:14:For streaming: tracks token-by-token timing to find bottleneck.
scripts/profile_pipeline.py:15:For concurrency: measures contention points under parallel load.
scripts/profile_pipeline.py:22:    python scripts/profile_pipeline.py --concurrency 1,5,10,20
scripts/profile_pipeline.py:81:    """Accumulated timing for a named pipeline stage."""
scripts/profile_pipeline.py:114:    """Complete timing profile for one TTS request."""
scripts/profile_pipeline.py:210:    """Profile a single non-streaming request with stage-level timing."""
scripts/profile_pipeline.py:347:    """Profile a single streaming request with per-token and per-decode timing."""
scripts/profile_pipeline.py:414:    # Per-token timing: time between consecutive yields from vLLM
scripts/profile_pipeline.py:422:        # Track inter-yield timing (approximation of per-token time)
scripts/profile_pipeline.py:530:    # Record inter-token timing stats
scripts/profile_pipeline.py:545:async def profile_concurrent(
scripts/profile_pipeline.py:552:    """Profile multiple concurrent requests."""
scripts/profile_pipeline.py:567:def print_concurrency_summary(profiles: List[RequestProfile], concurrency: int, stream: bool):
scripts/profile_pipeline.py:568:    """Print aggregated summary for concurrent profiles."""
scripts/profile_pipeline.py:576:    print(f"  CONCURRENCY={concurrency} ({mode}), {len(profiles)} requests")
scripts/profile_pipeline.py:591:    # Aggregate stage timing
scripts/profile_pipeline.py:673:    concurrency_levels = [int(x) for x in args.concurrency.split(",")]
scripts/profile_pipeline.py:676:    if not args.concurrency_only:
scripts/profile_pipeline.py:694:    for conc in concurrency_levels:
scripts/profile_pipeline.py:702:            logger.info(f"\nProfiling {conc} concurrent {'streaming' if stream_mode else 'non-streaming'} requests...")
scripts/profile_pipeline.py:703:            profiles = await profile_concurrent(runtime, conc, text, stream=stream_mode)
scripts/profile_pipeline.py:704:            print_concurrency_summary(profiles, conc, stream_mode)
scripts/profile_pipeline.py:714:    parser.add_argument("--concurrency", default="1,5,10,20", help="Concurrency levels (comma-separated)")
scripts/profile_pipeline.py:717:    parser.add_argument("--concurrency-only", action="store_true", help="Skip single-request profiles")
scripts/stress_test_local.py:11:- Higher concurrency ceiling (single-machine, no rate limits)
scripts/stress_test_local.py:12:- Real streaming TTFB measurement (byte-level timing)
scripts/stress_test_local.py:15:    # Full suite (ramps 1 → 5 → 10 → 20 → 50 → 100 concurrent)
scripts/stress_test_local.py:16:    python scripts/stress_test_local.py
scripts/stress_test_local.py:19:    python scripts/stress_test_local.py --quick
scripts/stress_test_local.py:22:    python scripts/stress_test_local.py --levels 1,5,10,20
scripts/stress_test_local.py:25:    python scripts/stress_test_local.py --stream-only
scripts/stress_test_local.py:28:    python scripts/stress_test_local.py --no-stream
scripts/stress_test_local.py:31:    python scripts/stress_test_local.py --url http://localhost:8080
scripts/stress_test_local.py:53:# Test texts at different lengths to stress different parts of the pipeline
scripts/stress_test_local.py:180:    concurrency: int
scripts/stress_test_local.py:266:    """Make a single TTS request with timing."""
scripts/stress_test_local.py:401:    concurrency: int,
scripts/stress_test_local.py:408:    level_name = f"{concurrency}c-{text_category}-{mode}"
scripts/stress_test_local.py:412:        concurrency=concurrency,
scripts/stress_test_local.py:418:    semaphore = asyncio.Semaphore(concurrency)
scripts/stress_test_local.py:440:        max_connections=concurrency + 10,
scripts/stress_test_local.py:441:        max_keepalive_connections=concurrency,
scripts/stress_test_local.py:473:    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}")
scripts/stress_test_local.py:545:async def run_stress_test(args):
scripts/stress_test_local.py:546:    """Run the full stress test suite."""
scripts/stress_test_local.py:564:    # Parse concurrency levels
scripts/stress_test_local.py:582:    warmup = await run_level(base_url, num_requests=2, concurrency=1, text_category="short", stream=False)
scripts/stress_test_local.py:598:            num_requests = max(conc * 3, 10)  # At least 3x concurrency or 10
scripts/stress_test_local.py:603:            print(f"\n>>> {conc} concurrent, {num_requests} total requests, text={text_cat}, stream={stream}")
scripts/stress_test_local.py:608:                concurrency=conc,
scripts/stress_test_local.py:640:            "concurrency": level.concurrency,
scripts/stress_test_local.py:665:    parser.add_argument("--levels", default="1,5,10,20,50", help="Comma-separated concurrency levels (default: 1,5,10,20,50)")
scripts/stress_test_local.py:670:    parser.add_argument("--output", default="stress_test_results.json", help="Output JSON path")
scripts/stress_test_local.py:677:    asyncio.run(run_stress_test(args))
scripts/validate_text_normalization.py:293:        "Class timings: 9:00 AM to 3:30 PM, Room #405",


cd /home/ubuntu/veenaModal && sed -n '1,260p' scripts/profile_pipeline.py


Output:
Chunk ID: 60304b
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 2188
Output:
#!/usr/bin/env python3
"""
Detailed Pipeline Profiler for Veena3 TTS.

Instruments every stage of the TTS pipeline with microsecond precision:
- Prompt building
- vLLM prefill (prompt encoding)
- vLLM decode (per-token generation)
- Token parsing (BiCodec extraction)
- BiCodec audio decode (GPU)
- Crossfade / audio post-processing
- WAV header creation

For streaming: tracks token-by-token timing to find bottleneck.
For concurrency: measures contention points under parallel load.

Usage:
    # Single request breakdown
    python scripts/profile_pipeline.py

    # Concurrency profiling (1, 5, 10, 20 parallel requests)
    python scripts/profile_pipeline.py --concurrency 1,5,10,20

    # Long text profiling
    python scripts/profile_pipeline.py --text long

    # Streaming profiling
    python scripts/profile_pipeline.py --stream
"""

from __future__ import annotations

import argparse
import asyncio
import logging
import os
import re
import sys
import time
import uuid
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

# === Path setup (same as local_server.py) ===
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
EXTERNAL_DIR = os.path.join(REPO_ROOT, "external")
for path in [os.path.join(EXTERNAL_DIR, "sparktts"), os.path.join(EXTERNAL_DIR, "AP-BWE"), REPO_ROOT]:
    if path not in sys.path:
        sys.path.insert(0, path)

os.environ["AUTH_BYPASS_MODE"] = "true"

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s | %(levelname)-8s | %(message)s",
    datefmt="%H:%M:%S",
)
logger = logging.getLogger("profiler")

# Test texts
TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This sentence tests basic text normalization and generation."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast long shadows "
        "over crowded streets, there lived an old bookkeeper named Margaret. She had spent "
        "forty years cataloging the stories of others, yet never found time to write her own. "
        "One rainy Tuesday afternoon, she discovered a leather-bound journal tucked behind "
        "a shelf of dusty encyclopedias. Its pages were blank, waiting."
    ),
}

# === Timing Utilities ===

@dataclass
class StageTimer:
    """Accumulated timing for a named pipeline stage."""
    name: str
    calls: int = 0
    total_ms: float = 0.0
    min_ms: float = float("inf")
    max_ms: float = 0.0
    samples: List[float] = field(default_factory=list)

    def record(self, ms: float):
        self.calls += 1
        self.total_ms += ms
        self.min_ms = min(self.min_ms, ms)
        self.max_ms = max(self.max_ms, ms)
        self.samples.append(ms)

    @property
    def avg_ms(self) -> float:
        return self.total_ms / self.calls if self.calls > 0 else 0

    def report_line(self) -> str:
        if self.calls == 0:
            return f"  {self.name:<35} (not called)"
        if self.calls == 1:
            return f"  {self.name:<35} {self.total_ms:>8.2f}ms"
        return (
            f"  {self.name:<35} {self.total_ms:>8.2f}ms total  "
            f"avg={self.avg_ms:.2f}ms  min={self.min_ms:.2f}ms  "
            f"max={self.max_ms:.2f}ms  calls={self.calls}"
        )

@dataclass
class RequestProfile:
    """Complete timing profile for one TTS request."""
    request_id: str
    text: str
    speaker: str
    stream: bool
    stages: Dict[str, StageTimer] = field(default_factory=dict)

    # Metadata filled during profiling
    total_ms: float = 0.0
    prompt_tokens: int = 0
    generated_tokens: int = 0
    semantic_tokens: int = 0
    global_tokens: int = 0
    audio_bytes: int = 0
    audio_seconds: float = 0.0

    # Streaming-specific
    ttfb_ms: float = 0.0
    chunks_emitted: int = 0
    token_generation_rates: List[float] = field(default_factory=list)  # tokens/sec samples
    decode_latencies: List[float] = field(default_factory=list)  # ms per BiCodec decode

    def stage(self, name: str) -> StageTimer:
        if name not in self.stages:
            self.stages[name] = StageTimer(name=name)
        return self.stages[name]

    def report(self) -> str:
        lines = [
            f"\n{'=' * 80}",
            f"  REQUEST PROFILE: {self.request_id}",
            f"  text={len(self.text)} chars, speaker={self.speaker}, stream={self.stream}",
            f"{'=' * 80}",
            f"  TOTAL: {self.total_ms:.1f}ms",
            f"  Generated: {self.generated_tokens} tokens ({self.semantic_tokens} semantic, {self.global_tokens} global)",
            f"  Audio: {self.audio_seconds:.2f}s ({self.audio_bytes} bytes)",
        ]
        if self.stream:
            lines.append(f"  TTFB: {self.ttfb_ms:.1f}ms, Chunks: {self.chunks_emitted}")
        if self.generated_tokens > 0 and self.total_ms > 0:
            tps = self.generated_tokens / (self.total_ms / 1000)
            lines.append(f"  Token gen rate: {tps:.1f} tok/s")
        if self.audio_seconds > 0:
            rtf = (self.total_ms / 1000) / self.audio_seconds
            lines.append(f"  RTF: {rtf:.3f}")

        lines.append(f"\n  {'STAGE BREAKDOWN':}")
        lines.append(f"  {'-' * 76}")

        # Order stages by total time (descending)
        sorted_stages = sorted(self.stages.values(), key=lambda s: s.total_ms, reverse=True)
        for s in sorted_stages:
            lines.append(s.report_line())

        # Token generation rate over time (streaming)
        if self.token_generation_rates:
            lines.append(f"\n  TOKEN GEN RATE (tok/s over time):")
            # Show first 5, middle, last 5
            rates = self.token_generation_rates
            if len(rates) <= 12:
                lines.append(f"    {' → '.join(f'{r:.0f}' for r in rates)}")
            else:
                first = ' → '.join(f'{r:.0f}' for r in rates[:5])
                last = ' → '.join(f'{r:.0f}' for r in rates[-5:])
                lines.append(f"    {first} → ... → {last}")
            import statistics
            lines.append(f"    avg={statistics.mean(rates):.0f}  min={min(rates):.0f}  max={max(rates):.0f}")

        # BiCodec decode latencies
        if self.decode_latencies:
            lines.append(f"\n  BICODEC DECODE LATENCIES (ms per call):")
            import statistics
            lines.append(
                f"    avg={statistics.mean(self.decode_latencies):.2f}  "
                f"min={min(self.decode_latencies):.2f}  "
                f"max={max(self.decode_latencies):.2f}  "
                f"calls={len(self.decode_latencies)}"
            )

        lines.append(f"{'=' * 80}")
        return "\n".join(lines)

# === Non-Streaming Profiler ===

async def profile_non_streaming(
    runtime,
    text: str,
    speaker: str = "lipakshi",
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
) -> RequestProfile:
    """Profile a single non-streaming request with stage-level timing."""
    from vllm import SamplingParams
    from veena3modal.core.constants import TRAINING_STOP_TOKEN_IDS, AUDIO_SAMPLE_RATE
    from veena3modal.audio.utils import add_wav_header

    profile = RequestProfile(
        request_id=f"prof-{uuid.uuid4().hex[:8]}",
        text=text,
        speaker=speaker,
        stream=False,
    )

    t_total_start = time.perf_counter()

    # === Stage 1: Prompt Building ===
    t0 = time.perf_counter()
    prompt = runtime.prompt_builder.build_prefix(speaker, text)
    profile.stage("1_prompt_build").record((time.perf_counter() - t0) * 1000)
    profile.prompt_tokens = len(runtime.model.tokenizer.encode(prompt, add_special_tokens=False))

    # === Stage 2: Sampling Config ===
    t0 = time.perf_counter()
    sampling_params = SamplingParams(
        temperature=temperature,
        top_k=top_k,
        top_p=top_p,
        max_tokens=max_tokens,
        repetition_penalty=repetition_penalty,
        stop=TRAINING_STOP_TOKEN_IDS,
        skip_special_tokens=False,
        seed=seed,
    )
    profile.stage("2_sampling_config").record((time.perf_counter() - t0) * 1000)

    # === Stage 3: vLLM Token Generation ===
    request_id = f"prof-{uuid.uuid4().hex[:8]}"
    t_gen_start = time.perf_counter()
    t_first_output = None
    iteration_count = 0
    prev_token_count = 0

    results_generator = runtime.model.engine.generate(
        prompt=prompt,
        sampling_params=sampling_params,
        request_id=request_id,
    )

    final_output = None
    async for request_output in results_generator:
        iteration_count += 1
        now = time.perf_counter()


cd /home/ubuntu/veenaModal && sed -n '260,760p' scripts/profile_pipeline.py