Output:
Chunk ID: 0c0965
Wall time: 0.0505 seconds
Process exited with code 0
Original token count: 2958
Output:
   260	        try:
   261	            from veena3modal.api.fastapi_app import set_model_version
   262	            set_model_version(model_version)
   263	        except ImportError:
   264	            pass
   265
   266	        return _runtime
   267
   268	    except Exception as e:
   269	        logger.error(f"❌ Failed to initialize TTS runtime: {e}")
   270	        import traceback
   271	        traceback.print_exc()
   272	        raise RuntimeError(f"TTS runtime initialization failed: {e}") from e
   273
   274
   275	def _precompute_speaker_globals(runtime: TTSRuntime) -> None:
   276	    """
   277	    Pre-compute and cache global tokens for all 12 speakers at startup.
   278
   279	    OPTIMIZATION: BiCodec streaming requires 32 "global tokens" before any audio can be emitted.
   280	    These encode speaker identity via FSQ quantization. Since we have only 12 fixed speakers,
   281	    we generate one short utterance per speaker at startup, capture the 32 global tokens,
   282	    and cache them. During streaming, we inject cached globals via build_prefix_with_globals(),
   283	    skipping the ~110ms global token pre-roll phase entirely.
   284
   285	    This runs synchronously at startup (adds ~5-10s to cold start, saves ~110ms per streaming TTFB).
   286	    """
   287	    import asyncio
   288	    import re
   289	    from vllm import SamplingParams
   290	    from veena3modal.core.constants import (
   291	        SPEAKER_MAP, TRAINING_STOP_TOKEN_IDS,
   292	        DEFAULT_TEMPERATURE, DEFAULT_TOP_K, DEFAULT_TOP_P,
   293	    )
   294
   295	    logger.info("Pre-computing global tokens for all speakers...")
   296	    t_start = time.time()
   297
   298	    test_text = "Hello, this is a voice test."
   299
   300	    sampling_params = SamplingParams(
   301	        temperature=DEFAULT_TEMPERATURE,
   302	        top_k=DEFAULT_TOP_K,
   303	        top_p=DEFAULT_TOP_P,
   304	        max_tokens=128,  # Only need ~60 tokens (32 global + some semantic)
   305	        stop=TRAINING_STOP_TOKEN_IDS,
   306	        skip_special_tokens=False,
   307	    )
   308
   309	    async def _capture_globals_for_speaker(speaker_name: str) -> Optional[List[int]]:
   310	        """Generate a short utterance and capture the 32 global tokens."""
   311	        prompt = runtime.prompt_builder.build_prefix(speaker_name, test_text)
   312	        request_id = f"warmup-{speaker_name}-{int(time.time() * 1000)}"
   313
   314	        final_output = None
   315	        async for request_output in runtime.model.engine.generate(
   316	            prompt=prompt,
   317	            sampling_params=sampling_params,
   318	            request_id=request_id,
   319	        ):
   320	            final_output = request_output
   321
   322	        if final_output is None:
   323	            return None
   324
   325	        generated_text = final_output.outputs[0].text
   326	        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
   327	        global_ids = [int(t) for t in global_matches]
   328
   329	        if len(global_ids) >= 32:
   330	            return global_ids[:32]
   331	        return None
   332
   333	    async def _run_all():
   334	        for speaker_name in SPEAKER_MAP.keys():
   335	            try:
   336	                global_ids = await _capture_globals_for_speaker(speaker_name)
   337	                if global_ids and len(global_ids) == 32:
   338	                    runtime.speaker_global_cache[speaker_name] = global_ids
   339	                    logger.info(f"  Cached globals for {speaker_name}")
   340	                else:
   341	                    logger.warning(f"  Failed to capture globals for {speaker_name}")
   342	            except Exception as e:
   343	                logger.warning(f"  Error caching globals for {speaker_name}: {e}")
   344
   345	    # Run async warmup in event loop
   346	    try:
   347	        loop = asyncio.get_event_loop()
   348	        if loop.is_running():
   349	            # Already in async context (unlikely at startup, but handle it)
   350	            import concurrent.futures
   351	            with concurrent.futures.ThreadPoolExecutor() as pool:
   352	                pool.submit(lambda: asyncio.run(_run_all())).result()
   353	        else:
   354	            loop.run_until_complete(_run_all())
   355	    except RuntimeError:
   356	        # No event loop exists yet
   357	        asyncio.run(_run_all())
   358
   359	    elapsed = (time.time() - t_start) * 1000
   360	    cached = len(runtime.speaker_global_cache)
   361	    total = len(SPEAKER_MAP)
   362	    logger.info(f"Speaker globals cached: {cached}/{total} speakers in {elapsed:.0f}ms")
   363
   364
   365	async def generate_speech(
   366	    text: str,
   367	    speaker: str,
   368	    temperature: float = 0.8,
   369	    top_k: int = 50,
   370	    top_p: float = 1.0,
   371	    max_tokens: int = 4096,
   372	    repetition_penalty: float = 1.05,
   373	    seed: Optional[int] = None,
   374	    output_sample_rate: str = "16khz",
   375	) -> Tuple[Optional[bytes], Dict[str, Any]]:
   376	    """
   377	    Generate speech audio (non-streaming).
   378
   379	    Args:
   380	        text: Text to synthesize (already normalized)
   381	        speaker: Internal speaker name (resolved)
   382	        temperature: Sampling temperature
   383	        top_k: Top-k sampling
   384	        top_p: Nucleus sampling
   385	        max_tokens: Maximum tokens to generate
   386	        repetition_penalty: Repetition penalty
   387	        seed: Random seed for reproducibility
   388	        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)
   389
   390	    Returns:
   391	        Tuple of (audio_bytes, metrics_dict)
   392	        audio_bytes: WAV audio bytes (16kHz or 48kHz, 16-bit PCM) or None if failed
   393	        metrics_dict: Dictionary with timing metrics
   394
   395	    Raises:
   396	        RuntimeError: If runtime not initialized
   397	    """
   398	    if not is_initialized():
   399	        raise RuntimeError("TTS runtime not initialized")
   400
   401	    runtime = get_runtime()
   402	    metrics = {
   403	        "ttfb_ms": 0,
   404	        "generation_ms": 0,
   405	        "tokens_generated": 0,
   406	        "audio_duration_seconds": 0.0,
   407	        "sr_applied": False,
   408	        "output_sample_rate": 16000,
   409	    }
   410
   411	    active_requests = _with_inflight_delta(1)
   412	    start_time = time.time()
   413
   414	    pipeline_timings = {}
   415
   416	    try:
   417	        logger.info(
   418	            "tts_runtime_nonstream_start",
   419	            extra={
   420	                "speaker": speaker,
   421	                "text_length": len(text),
   422	                "request_inflight": active_requests,
   423	                "temperature": temperature,
   424	                "top_k": top_k,
   425	                "top_p": top_p,
   426	                "max_tokens": max_tokens,
   427	            },
   428	        )
   429
   430	        gen_kwargs = dict(
   431	            speaker=speaker,
   432	            text=text,
   433	            temperature=temperature,
   434	            top_k=top_k,
   435	            top_p=top_p,
   436	            max_tokens=max_tokens,
   437	            repetition_penalty=repetition_penalty,
   438	            seed=seed,
   439	        )
   440
   441	        if _timing_enabled():
   442	            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
   443	                **gen_kwargs,
   444	                return_timing=True,
   445	            )
   446	        else:
   447	            audio_bytes = await runtime.pipeline.generate_speech_indic(**gen_kwargs)
   448
   449	        generation_time = time.time() - start_time
   450	        metrics["generation_ms"] = int(generation_time * 1000)
   451	        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time
   452
   453	        if pipeline_timings:
   454	            metrics.update({
   455	                "llm_token_total": pipeline_timings.get("token_total", 0),
   456	                "llm_batch_count": pipeline_timings.get("batch_count", 0),
   457	                "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
   458	                "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
   459	                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
   460	                "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
   461	                "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
   462	                "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
   463	                "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
   464	            })
   465
   466	        if audio_bytes:
   467	            # Apply super-resolution if requested and available
   468	            sample_rate = 16000
   469	            logger.info(f"SR check: output_sample_rate={output_sample_rate!r}, "
   470	                       f"sr_service={runtime.sr_service is not None}, "
   471	                       f"is_loaded={runtime.sr_service.is_loaded if runtime.sr_service else False}")
   472
   473	            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
   474	                try:
   475	                    sr_start = time.time()
   476	                    logger.info(f"Applying super-resolution to {len(audio_bytes)} bytes...")
   477	                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
   478	                    sr_time = (time.time() - sr_start) * 1000
   479	                    metrics["sr_ms"] = int(sr_time)
   480	                    metrics["sr_applied"] = True
   481	                    sample_rate = 48000
   482	                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
   483	                except Exception as e:
   484	                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
   485	                    import traceback
   486	                    traceback.print_exc()
   487
   488	            metrics["output_sample_rate"] = sample_rate
   489
   490	            # Calculate audio duration
   491	                audio_duration = (len(audio_bytes) - 44) / (sample_rate * 2)  # -44 for WAV header
   492	                metrics["audio_duration_seconds"] = max(0.0, audio_duration)
   493
   494	        if _timing_enabled():
   495	            logger.info("tts_runtime_nonstream_complete", extra=metrics)
   496
   497	        return audio_bytes, metrics
   498
   499	    except Exception as e:
   500	        logger.error(f"Speech generation failed: {e}")
   501	        raise
   502	    finally:
   503	        active_requests = _with_inflight_delta(-1)
   504	        if _timing_enabled():
   505	            logger.info(
   506	                "tts_runtime_nonstream_inflight",
   507	                extra={"active_requests": active_requests},
   508	            )
   509
   510
   511	def _apply_super_resolution(audio_bytes: bytes, sr_service) -> bytes:
   512	    """
   513	    Apply super-resolution to audio bytes.
   514
   515	    Args:
   516	        audio_bytes: WAV audio at 16kHz
   517	        sr_service: SuperResolutionService instance
   518
   519	    Returns:
   520	        WAV audio at 48kHz


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


nl -ba /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py | sed -n '520,980p'


Output:
Chunk ID: c09348
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 5073
Output:
   520	        WAV audio at 48kHz
   521	    """
   522	    import numpy as np
   523	    import struct
   524	    import torch
   525
   526	    # Parse WAV header and extract PCM data
   527	    if len(audio_bytes) < 44:
   528	        raise ValueError("Invalid WAV data")
   529
   530	    # Extract PCM data (skip 44-byte WAV header)
   531	    pcm_data = np.frombuffer(audio_bytes[44:], dtype=np.int16)
   532
   533	    # Convert to float32 for SR model
   534	    audio_float = pcm_data.astype(np.float32) / 32768.0
   535
   536	    # Convert numpy to torch tensor for process_chunk
   537	    audio_tensor = torch.from_numpy(audio_float)
   538
   539	    # Apply super-resolution using process_chunk (16kHz -> 48kHz)
   540	    # process_chunk expects [batch, samples] or [samples] and returns [batch, samples]
   541	    upsampled_tensor = sr_service.process_chunk(audio_tensor)
   542
   543	    # Convert back to numpy, squeeze batch dim if present
   544	    upsampled = upsampled_tensor.squeeze().cpu().numpy()
   545
   546	    # Convert back to int16
   547	    upsampled_int16 = np.clip(upsampled * 32768.0, -32768, 32767).astype(np.int16)
   548
   549	    # Create new WAV header for 48kHz
   550	    sample_rate = 48000
   551	    num_channels = 1
   552	    bits_per_sample = 16
   553	    byte_rate = sample_rate * num_channels * bits_per_sample // 8
   554	    block_align = num_channels * bits_per_sample // 8
   555	    data_size = len(upsampled_int16) * 2
   556
   557	    wav_header = struct.pack(
   558	        '<4sI4s4sIHHIIHH4sI',
   559	        b'RIFF',
   560	        36 + data_size,
   561	        b'WAVE',
   562	        b'fmt ',
   563	        16,  # fmt chunk size
   564	        1,   # PCM format
   565	        num_channels,
   566	        sample_rate,
   567	        byte_rate,
   568	        block_align,
   569	        bits_per_sample,
   570	        b'data',
   571	        data_size,
   572	    )
   573
   574	    return wav_header + upsampled_int16.tobytes()
   575
   576
   577	async def generate_speech_chunked(
   578	    text: str,
   579	    speaker: str,
   580	    temperature: float = 0.8,
   581	    top_k: int = 50,
   582	    top_p: float = 1.0,
   583	    max_tokens: int = 4096,
   584	    repetition_penalty: float = 1.05,
   585	    seed: Optional[int] = None,
   586	    sample_rate: int = 16000,
   587	    output_sample_rate: str = "16khz",
   588	) -> Tuple[Optional[bytes], Dict[str, Any]]:
   589	    """
   590	    Generate speech with automatic text chunking for long inputs.
   591
   592	    Uses LongTextProcessor to split text and stitch audio.
   593
   594	    Args:
   595	        text: Text to synthesize
   596	        speaker: Internal speaker name
   597	        ... (same as generate_speech)
   598	        sample_rate: Internal generation sample rate (always 16000)
   599	        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)
   600
   601	    Returns:
   602	        Tuple of (audio_bytes, metrics_dict)
   603	    """
   604	    if not is_initialized():
   605	        raise RuntimeError("TTS runtime not initialized")
   606
   607	    runtime = get_runtime()
   608	    metrics = {
   609	        "ttfb_ms": 0,
   610	        "generation_ms": 0,
   611	        "chunks_processed": 0,
   612	        "audio_duration_seconds": 0.0,
   613	        "text_chunked": False,
   614	        "sr_applied": False,
   615	        "output_sample_rate": 16000,
   616	    }
   617
   618	    active_requests = _with_inflight_delta(1)
   619	    start_time = time.time()
   620
   621	    try:
   622	        if _timing_enabled():
   623	            logger.info(
   624	                "tts_runtime_chunking_start",
   625	                extra={
   626	                    "speaker": speaker,
   627	                    "text_length": len(text),
   628	                    "request_inflight": active_requests,
   629	                    "chunking": True,
   630	                },
   631	            )
   632	        # Import long text processor
   633	        from veena3modal.processing.long_text_processor import LongTextProcessor
   634
   635	        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
   636
   637	        # Check if chunking is needed
   638	        if long_processor.should_chunk(text):
   639	            metrics["text_chunked"] = True
   640	            audio_bytes = await long_processor.generate_with_chunking(
   641	                text=text,
   642	                speaker=speaker,
   643	                temperature=temperature,
   644	                top_k=top_k,
   645	                top_p=top_p,
   646	                max_tokens=max_tokens,
   647	                repetition_penalty=repetition_penalty,
   648	                seed=seed,
   649	                sample_rate=sample_rate,
   650	            )
   651	        else:
   652	            # Short text: direct generation
   653	            audio_bytes = await runtime.pipeline.generate_speech_indic(
   654	                speaker=speaker,
   655	                text=text,
   656	                temperature=temperature,
   657	                top_k=top_k,
   658	                top_p=top_p,
   659	                max_tokens=max_tokens,
   660	                repetition_penalty=repetition_penalty,
   661	                seed=seed,
   662	            )
   663
   664	        generation_time = time.time() - start_time
   665	        metrics["generation_ms"] = int(generation_time * 1000)
   666	        metrics["ttfb_ms"] = metrics["generation_ms"]
   667
   668	        if audio_bytes:
   669	            # Apply super-resolution if requested and available
   670	            final_sample_rate = 16000
   671	            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
   672	                try:
   673	                    sr_start = time.time()
   674	                    logger.info(f"Applying super-resolution to chunked audio ({len(audio_bytes)} bytes)...")
   675	                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
   676	                    sr_time = (time.time() - sr_start) * 1000
   677	                    metrics["sr_ms"] = int(sr_time)
   678	                    metrics["sr_applied"] = True
   679	                    final_sample_rate = 48000
   680	                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
   681	                except Exception as e:
   682	                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
   683	                    import traceback
   684	                    traceback.print_exc()
   685
   686	            metrics["output_sample_rate"] = final_sample_rate
   687	            audio_duration = (len(audio_bytes) - 44) / (final_sample_rate * 2)
   688	            metrics["audio_duration_seconds"] = max(0.0, audio_duration)
   689
   690	        if _timing_enabled():
   691	            metrics["active_requests"] = active_requests - 1
   692	            logger.info("tts_runtime_chunking_complete", extra=metrics)
   693	        return audio_bytes, metrics
   694
   695	    except Exception as e:
   696	        logger.error(f"Chunked speech generation failed: {e}")
   697	        raise
   698	    finally:
   699	        _with_inflight_delta(-1)
   700
   701
   702	from typing import AsyncGenerator
   703
   704
   705	async def generate_speech_streaming(
   706	    text: str,
   707	    speaker: str,
   708	    temperature: float = 0.8,
   709	    top_k: int = 50,
   710	    top_p: float = 1.0,
   711	    max_tokens: int = 4096,
   712	    repetition_penalty: float = 1.05,
   713	    seed: Optional[int] = None,
   714	    enable_chunking: bool = True,
   715	    request_id: Optional[str] = None,
   716	) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
   717	    """
   718	    Generate speech audio with true streaming (yields chunks as they're generated).
   719
   720	    This is the core streaming implementation for M4.
   721	    First yield includes WAV header + first PCM chunk.
   722	    Subsequent yields are raw PCM chunks.
   723
   724	    Args:
   725	        text: Text to synthesize (already normalized)
   726	        speaker: Internal speaker name (resolved)
   727	        temperature: Sampling temperature
   728	        top_k: Top-k sampling
   729	        top_p: Nucleus sampling
   730	        max_tokens: Maximum tokens to generate
   731	        repetition_penalty: Repetition penalty
   732	        seed: Random seed for reproducibility
   733	        enable_chunking: Enable text chunking for long inputs with voice consistency
   734
   735	    Yields:
   736	        Tuple of (audio_bytes, metrics_dict)
   737	        - First yield: WAV header (44 bytes) prepended to first PCM chunk
   738	        - Subsequent yields: Raw PCM chunks (int16, 16kHz)
   739	        - Final yield: metrics_dict has final timing info
   740
   741	    Raises:
   742	        RuntimeError: If runtime not initialized or streaming pipeline unavailable
   743	    """
   744	    if not is_initialized():
   745	        raise RuntimeError("TTS runtime not initialized")
   746
   747	    runtime = get_runtime()
   748
   749	    if runtime.streaming_pipeline is None:
   750	        raise RuntimeError("Streaming pipeline not available")
   751
   752	    # Import audio utils for WAV header
   753	    from veena3modal.audio.utils import create_wav_header
   754
   755	    # Metrics tracking
   756	    metrics = {
   757	        "ttfb_ms": 0,
   758	        "chunks_sent": 0,
   759	        "total_bytes": 0,
   760	        "audio_duration_seconds": 0.0,
   761	        "text_chunked": False,
   762	    }
   763
   764	    start_time = time.time()
   765	    first_chunk_time = None
   766	    total_pcm_bytes = 0
   767	    sample_rate = 16000  # BiCodec sample rate
   768	    active_requests = _with_inflight_delta(1)
   769	    if _timing_enabled():
   770	        logger.info(
   771	            "tts_runtime_stream_start",
   772	            extra={
   773	                "speaker": speaker,
   774	                "text_length": len(text),
   775	                "request_inflight": active_requests,
   776	                "chunking": enable_chunking,
   777	                "request_id": request_id,
   778	            },
   779	        )
   780
   781	    try:
   782	        # Check if we need text chunking
   783	        from veena3modal.processing.long_text_processor import LongTextProcessor
   784	        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
   785	        needs_chunking = enable_chunking and long_processor.should_chunk(text)
   786
   787	        if needs_chunking:
   788	            # Chunked streaming with voice consistency (global token caching)
   789	            metrics["text_chunked"] = True
   790	            async for audio_chunk, chunk_metrics in _stream_chunked_text(
   791	                runtime,
   792	                long_processor,
   793	                text,
   794	                speaker,
   795	                temperature,
   796	                top_k,
   797	                top_p,
   798	                max_tokens,
   799	                repetition_penalty,
   800	                seed,
   801	                sample_rate,
   802	                request_id,
   803	            ):
   804	                if first_chunk_time is None:
   805	                    first_chunk_time = time.time()
   806	                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
   807	                    # Create streaming WAV header (size=0 for unknown length)
   808	                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
   809	                    audio_chunk = wav_header + audio_chunk
   810
   811	                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
   812	                metrics["chunks_sent"] += 1
   813	                metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
   814	                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
   815	                if chunk_metrics:
   816	                    metrics.update(chunk_metrics)
   817
   818	                yield audio_chunk, metrics
   819	        else:
   820	            # Simple streaming (no text chunking)
   821	            # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
   822	            # use continuation mode to skip the ~110ms global token pre-roll.
   823	            # The model jumps straight to semantic token generation.
   824	            cached_globals = runtime.speaker_global_cache.get(speaker)
   825
   826	            if cached_globals:
   827	                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
   828	                    speaker=speaker,
   829	                    text=text,
   830	                    global_ids=cached_globals,
   831	                    temperature=temperature,
   832	                    top_k=top_k,
   833	                    top_p=top_p,
   834	                    max_tokens=max_tokens,
   835	                    repetition_penalty=repetition_penalty,
   836	                    seed=seed,
   837	                    request_id=request_id,
   838	                )
   839	            else:
   840	                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
   841	                    speaker=speaker,
   842	                    text=text,
   843	                    temperature=temperature,
   844	                    top_k=top_k,
   845	                    top_p=top_p,
   846	                    max_tokens=max_tokens,
   847	                    repetition_penalty=repetition_penalty,
   848	                    seed=seed,
   849	                    request_id=request_id,
   850	                )
   851
   852	            async for audio_chunk in stream_gen:
   853	                if first_chunk_time is None:
   854	                    first_chunk_time = time.time()
   855	                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
   856	                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
   857	                    audio_chunk = wav_header + audio_chunk
   858
   859	                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
   860	                metrics["chunks_sent"] += 1
   861	                metrics["total_bytes"] = total_pcm_bytes + 44
   862	                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
   863	                yield audio_chunk, metrics
   864	    finally:
   865	        active_requests = _with_inflight_delta(-1)
   866	        if _timing_enabled():
   867	            logger.info(
   868	                "tts_runtime_stream_end",
   869	                extra={
   870	                    "speaker": speaker,
   871	                    "request_id": request_id,
   872	                    "request_inflight": active_requests,
   873	                    "chunks_sent": metrics.get("chunks_sent", 0),
   874	                },
   875	            )
   876
   877
   878	async def _stream_chunked_text(
   879	    runtime: TTSRuntime,
   880	    long_processor,
   881	    text: str,
   882	    speaker: str,
   883	    temperature: float,
   884	    top_k: int,
   885	    top_p: float,
   886	    max_tokens: int,
   887	    repetition_penalty: float,
   888	    seed: Optional[int],
   889	    sample_rate: int,
   890	    request_id: Optional[str] = None,
   891	) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
   892	    """
   893	    Internal helper: stream chunked text with voice consistency.
   894
   895	    Uses global token caching from first chunk to maintain voice across chunks.
   896	    """
   897	    from veena3modal.audio.crossfade import crossfade_bytes_int16
   898
   899	    # Chunk text using the long text processor's chunker
   900	    chunks = long_processor.chunk_text(text)
   901
   902	    if not chunks:
   903	        return
   904
   905	    captured_globals: Optional[List[int]] = None
   906	    previous_chunk_tail: Optional[bytes] = None
   907	    chunk_metrics = {"chunks_processed": 0}
   908
   909	    for i, chunk_text in enumerate(chunks):
   910	        chunk_metrics["chunks_processed"] = i + 1
   911
   912	        if i == 0:
   913	            # First chunk: capture global tokens for voice consistency
   914	            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
   915	                speaker=speaker,
   916	                text=chunk_text,
   917	                temperature=temperature,
   918	                top_k=top_k,
   919	                top_p=top_p,
   920	                max_tokens=max_tokens,
   921	                repetition_penalty=repetition_penalty,
   922	                seed=seed,
   923	                request_id=request_id,
   924	            ):
   925	                if captured_globals is None and global_ids:
   926	                    captured_globals = global_ids
   927
   928	                # Crossfade with previous tail (inter-chunk stitching)
   929	                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   930	                    previous_chunk_tail,
   931	                    audio_bytes,
   932	                    sample_rate_hz=sample_rate,
   933	                    crossfade_ms=50,
   934	                )
   935
   936	                if to_emit:
   937	                    yield to_emit, chunk_metrics
   938	        else:
   939	            # Continuation chunks: use captured globals for voice consistency
   940	            if captured_globals is None:
   941	                logger.warning(f"No captured globals for chunk {i+1}, using regular streaming")
   942	                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
   943	                    speaker=speaker,
   944	                    text=chunk_text,
   945	                    temperature=temperature,
   946	                    top_k=top_k,
   947	                    top_p=top_p,
   948	                    max_tokens=max_tokens,
   949	                    repetition_penalty=repetition_penalty,
   950	                    seed=seed,
   951	                    request_id=request_id,
   952	                ):
   953	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   954	                        previous_chunk_tail,
   955	                        audio_bytes,
   956	                        sample_rate_hz=sample_rate,
   957	                        crossfade_ms=50,
   958	                    )
   959	                    if to_emit:
   960	                        yield to_emit, chunk_metrics
   961	            else:
   962	                # Use continuation method with cached globals
   963	                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
   964	                    speaker=speaker,
   965	                    text=chunk_text,
   966	                    global_ids=captured_globals,
   967	                    temperature=temperature,
   968	                    top_k=top_k,
   969	                    top_p=top_p,
   970	                    max_tokens=max_tokens,
   971	                    repetition_penalty=repetition_penalty,
   972	                    seed=seed,
   973	                    request_id=request_id,
   974	                ):
   975	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   976	                        previous_chunk_tail,
   977	                        audio_bytes,
   978	                        sample_rate_hz=sample_rate,
   979	                        crossfade_ms=50,
   980	                    )


Summary: auto


nl -ba /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py | sed -n '980,1140p'


Output:
Chunk ID: 221593
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 67
Output:
   980	                    )
   981	                    if to_emit:
   982	                        yield to_emit, chunk_metrics
   983
   984	    # Flush remaining tail
   985	    if previous_chunk_tail:
   986	        yield previous_chunk_tail, chunk_metrics


nl -ba /home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py | sed -n '490,620p'


Output:
Chunk ID: c2f470
Wall time: 0.0502 seconds
Process exited with code 0
Original token count: 1657
Output:
   490	                content={
   491	                    "error": {
   492	                        "code": "INTERNAL_ERROR",
   493	                        "message": str(e),
   494	                        "request_id": request_id,
   495	                    }
   496	                },
   497	                headers={"X-Request-ID": request_id},
   498	            )
   499
   500	    async def _handle_streaming_request(req, request_id: str, text: str, speaker: str, start_time: float):
   501	        """
   502	        Handle streaming TTS request.
   503
   504	        Returns a StreamingResponse that yields WAV header + PCM chunks.
   505	        True streaming: first bytes sent ASAP before full audio is generated.
   506	        """
   507	        from fastapi.responses import StreamingResponse
   508	        from veena3modal.api.schemas import AudioFormat
   509	        from veena3modal.services import tts_runtime
   510
   511	        # Check if streaming pipeline is available
   512	        runtime = tts_runtime.get_runtime()
   513	        if runtime is None or runtime.streaming_pipeline is None:
   514	            return JSONResponse(
   515	                status_code=503,
   516	                content={
   517	                    "error": {
   518	                        "code": "STREAMING_UNAVAILABLE",
   519	                        "message": "Streaming pipeline not initialized.",
   520	                        "request_id": request_id,
   521	                    }
   522	                },
   523	                headers={"X-Request-ID": request_id},
   524	            )
   525
   526	        # Non-WAV formats not implemented yet (M5)
   527	        if req.format != AudioFormat.WAV.value:
   528	            return JSONResponse(
   529	                status_code=501,
   530	                content={
   531	                    "error": {
   532	                        "code": "FORMAT_NOT_IMPLEMENTED",
   533	                        "message": f"Streaming format '{req.format}' not yet implemented. Use format=wav.",
   534	                        "request_id": request_id,
   535	                    }
   536	                },
   537	                headers={"X-Request-ID": request_id},
   538	            )
   539
   540	        # Set request context for structured logging
   541	        set_request_context(request_id=request_id)
   542
   543	        # Create async generator for streaming response
   544	        async def audio_stream_generator():
   545	            """Yields audio chunks as they're generated."""
   546	            final_metrics = {}
   547	            first_chunk_sent = False
   548	            chunks_count = 0
   549	            try:
   550	                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
   551	                    text=text,
   552	                    speaker=speaker,
   553	                    temperature=req.temperature,
   554	                    top_k=req.top_k,
   555	                    top_p=req.top_p,
   556	                    max_tokens=req.max_tokens,
   557	                    repetition_penalty=req.repetition_penalty,
   558	                    seed=req.seed,
   559	                    enable_chunking=req.chunking,
   560	                ):
   561	                    final_metrics = metrics
   562	                    chunks_count += 1
   563
   564	                    # Fire-and-forget sentence storage after first chunk (don't block TTFB)
   565	                    if not first_chunk_sent:
   566	                        first_chunk_sent = True
   567	                        ttfb_ms = metrics.get("ttfb_ms", 0)
   568
   569	                        # Log first audio emitted (TTFB marker)
   570	                        log_first_audio_emitted(
   571	                            request_id=request_id,
   572	                            ttfb_ms=ttfb_ms,
   573	                            chunk_size_bytes=len(audio_chunk),
   574	                        )
   575	                        record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)
   576
   577	                        sentence_store = _get_sentence_store()
   578	                        sentence_store.store_fire_and_forget(
   579	                            request_id=request_id,
   580	                            text=text,
   581	                            speaker=speaker,
   582	                            stream=True,
   583	                            format=req.format,
   584	                            temperature=req.temperature,
   585	                            top_k=req.top_k,
   586	                            top_p=req.top_p,
   587	                            max_tokens=req.max_tokens,
   588	                            repetition_penalty=req.repetition_penalty,
   589	                            seed=req.seed,
   590	                            text_chunked=metrics.get("text_chunked", False),
   591	                            ttfb_ms=ttfb_ms,
   592	                            # audio_duration_seconds not known yet for streaming
   593	                        )
   594
   595	                    yield audio_chunk
   596
   597	                # Stream completed - log metrics
   598	                total_time = time.time() - start_time
   599	                audio_duration = final_metrics.get("audio_duration_seconds", 0)
   600	                rtf = total_time / audio_duration if audio_duration > 0 else 0
   601
   602	                log_request_completed(
   603	                    request_id=request_id,
   604	                    status_code=200,
   605	                    total_duration_ms=int(total_time * 1000),
   606	                    audio_duration_seconds=audio_duration,
   607	                    rtf=rtf,
   608	                    chunks_sent=chunks_count,
   609	                )
   610	                metrics_request_completed(
   611	                    status_code=200,
   612	                    duration_seconds=total_time,
   613	                    speaker=speaker,
   614	                    stream=True,
   615	                )
   616	                record_rtf(rtf=rtf, speaker=speaker)
   617	                record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
   618	                record_chunks_sent(chunks=chunks_count, speaker=speaker)
   619	                clear_request_context()
   620	            except Exception as e:


nl -ba /home/ubuntu/veenaModal/veena3modal/api/websocket_handler.py | sed -n '312,390p'


Output:
Chunk ID: f7e428
Wall time: 0.0517 seconds
Process exited with code 0
Original token count: 934
Output:
   312	                            websocket.receive_text(),
   313	                            timeout=0.1
   314	                        )
   315	                        data = json.loads(msg)
   316	                        if data.get("event") == "cancel":
   317	                            cancelled = True
   318	                            logger.info(f"Client requested cancel: {request_id}")
   319	                            break
   320	                        elif data.get("event") == "ping":
   321	                            await websocket.send_text(json.dumps({"event": "pong"}))
   322	                    except asyncio.TimeoutError:
   323	                        continue
   324	                    except Exception:
   325	                        break
   326	            except Exception:
   327	                pass
   328
   329	        cancel_task = asyncio.create_task(listen_for_cancel())
   330
   331	        try:
   332	            # Apply text normalization if enabled
   333	            text = req.text
   334	            if req.normalize:
   335	                try:
   336	                    from veena3modal.processing.text_normalizer import normalize_text
   337	                    text = normalize_text(text)
   338	                except ImportError:
   339	                    pass  # Normalizer not available
   340
   341	            # Stream audio chunks
   342	            async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
   343	                text=text,
   344	                speaker=speaker,
   345	                temperature=req.temperature,
   346	                top_k=req.top_k,
   347	                top_p=req.top_p,
   348	                max_tokens=req.max_tokens,
   349	                repetition_penalty=req.repetition_penalty,
   350	                seed=req.seed,
   351	                enable_chunking=req.chunking,
   352	            ):
   353	                if cancelled:
   354	                    logger.info(f"Generation cancelled: {request_id}")
   355	                    break
   356
   357	                final_metrics = metrics
   358	                chunks_sent += 1
   359	                total_bytes += len(audio_chunk)
   360
   361	                # Record TTFB on first chunk
   362	                if first_chunk_time is None:
   363	                    first_chunk_time = time.time()
   364	                    ttfb_ms = int((first_chunk_time - start_time) * 1000)
   365	                    log_first_audio_emitted(
   366	                        request_id=request_id,
   367	                        ttfb_ms=ttfb_ms,
   368	                        chunk_size_bytes=len(audio_chunk),
   369	                    )
   370	                    record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)
   371
   372	                # Send audio as binary frame
   373	                await websocket.send_bytes(audio_chunk)
   374
   375	                # Send progress every 10 chunks
   376	                if chunks_sent % 10 == 0:
   377	                    await websocket.send_text(create_progress_message(
   378	                        chunks_sent=chunks_sent,
   379	                        bytes_sent=total_bytes,
   380	                        elapsed_ms=int((time.time() - start_time) * 1000),
   381	                    ))
   382
   383	        finally:
   384	            cancel_task.cancel()
   385	            try:
   386	                await cancel_task
   387	            except asyncio.CancelledError:
   388	                pass
   389
   390	        # === SEND COMPLETION ===


nl -ba /home/ubuntu/veenaModal/veena3modal/core/pipeline.py | sed -n '1,220p'


Output:
Chunk ID: f182e4
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 2263
Output:
     1	"""
     2	Spark TTS Generation Pipeline
     3
     4	End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
     5	Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
     6	"""
     7
     8	import asyncio
     9	import logging
    10	import os
    11	from statistics import mean, median
    12	from typing import Optional, List, Dict, Any, Tuple
    13
    14	from vllm import SamplingParams
    15	import torch
    16
    17	logger = logging.getLogger(__name__)
    18
    19	from veena3modal.core.constants import (
    20	    TRAINING_STOP_TOKEN_IDS,
    21	    DEFAULT_TEMPERATURE,
    22	    DEFAULT_TOP_K,
    23	    DEFAULT_TOP_P,
    24	    DEFAULT_MAX_TOKENS,
    25	    DEFAULT_SEED,
    26	    AUDIO_SAMPLE_RATE,
    27	)
    28
    29
    30	def _detailed_timing_enabled() -> bool:
    31	    """
    32	    Return True when benchmark logs are enabled.
    33
    34	    Controlled by environment variable:
    35	      VEENA3_TIMING_LOGS=true|1
    36	    """
    37	    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}
    38
    39
    40	class SparkTTSPipeline:
    41	    """
    42	    End-to-end TTS pipeline for Spark TTS with BiCodec.
    43
    44	    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    45	    """
    46
    47	    def __init__(
    48	        self,
    49	        model,
    50	        prompt_builder,
    51	        bicodec_decoder,
    52	    ):
    53	        """
    54	        Initialize pipeline.
    55
    56	        Args:
    57	            model: SparkTTS Model instance
    58	            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
    59	            bicodec_decoder: BiCodecDecoder instance
    60	        """
    61	        self.model = model
    62	        self.prompt_builder = prompt_builder
    63	        self.bicodec_decoder = bicodec_decoder
    64
    65	        print(f"🚀 SparkTTSPipeline initialized")
    66
    67	    async def generate_speech(
    68	        self,
    69	        speaker: str,
    70	        text: str,
    71	        temperature: float = DEFAULT_TEMPERATURE,
    72	        top_k: int = DEFAULT_TOP_K,
    73	        top_p: float = DEFAULT_TOP_P,
    74	        max_tokens: int = DEFAULT_MAX_TOKENS,
    75	        repetition_penalty: float = 1.05,
    76	        seed: Optional[int] = None,
    77	        return_timing: bool = False,
    78	    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
    79	        """
    80	        Generate speech audio (non-streaming) using Spark TTS.
    81
    82	        NOTE: This method signature changed from description-based to speaker-based
    83	        to align with Spark TTS architecture.
    84
    85	        Args:
    86	            speaker: Speaker name (one of 12 predefined speakers)
    87	            text: Text to synthesize (with optional [emotion] tags)
    88	            temperature: Sampling temperature (default: 0.8 for Spark TTS)
    89	            top_k: Top-k sampling (default: 50)
    90	            top_p: Nucleus sampling (default: 1.0)
    91	            max_tokens: Max BiCodec tokens to generate (default: 2048)
    92	            seed: Random seed for reproducibility
    93	            return_timing: If true, returns (audio_bytes, metrics_dict)
    94
    95	        Returns:
    96	            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
    97	            If return_timing is true, returns tuple(audio_bytes, metrics).
    98	        """
    99	        # Build prompt using Spark TTS format
   100	        prompt = self.prompt_builder.build_prefix(speaker, text)
   101
   102	        # Configure sampling for Spark TTS
   103	        sampling_params = SamplingParams(
   104	            temperature=temperature,
   105	            top_k=top_k,
   106	            top_p=top_p,
   107	            max_tokens=max_tokens,
   108	            repetition_penalty=repetition_penalty,
   109	            stop=TRAINING_STOP_TOKEN_IDS,
   110	            skip_special_tokens=False,
   111	            seed=seed,
   112	        )
   113
   114	        # Use vLLM engine to generate
   115	        import uuid
   116	        request_id = f"req-{uuid.uuid4().hex[:12]}"
   117
   118	        emit_timing = return_timing or _detailed_timing_enabled()
   119	        results_generator = self.model.engine.generate(
   120	            prompt=prompt,
   121	            sampling_params=sampling_params,
   122	            request_id=request_id,
   123	        )
   124
   125	        # Collect all outputs and track generation timing
   126	        final_output = None
   127	        token_batches: List[int] = []
   128	        batch_wall_ms: List[float] = []
   129	        total_token_ids = 0
   130	        first_token_ms: Optional[float] = None
   131	        t_start = asyncio.get_event_loop().time()
   132
   133	        iterator = results_generator.__aiter__()
   134	        while True:
   135	            wall_start = asyncio.get_event_loop().time()
   136	            start_event = None
   137	            end_event = None
   138	            if torch.cuda.is_available():
   139	                torch.cuda.synchronize()
   140	                start_event = torch.cuda.Event(enable_timing=True)
   141	                end_event = torch.cuda.Event(enable_timing=True)
   142	                start_event.record()
   143
   144	            try:
   145	                request_output = await iterator.__anext__()
   146	            except StopAsyncIteration:
   147	                break
   148
   149	            wall_end = asyncio.get_event_loop().time()
   150	            batch_ms = (wall_end - wall_start) * 1000
   151	            batch_wall_ms.append(batch_ms)
   152
   153	            if start_event is not None and end_event is not None:
   154	                end_event.record()
   155	                torch.cuda.synchronize()
   156	                batch_gpu_ms = start_event.elapsed_time(end_event)
   157	            else:
   158	                batch_gpu_ms = None
   159
   160	            total_generated = len(request_output.outputs[0].token_ids)
   161	            new_tokens = max(0, total_generated - total_token_ids)
   162	            total_token_ids = total_generated
   163	            token_batches.append(new_tokens)
   164
   165	            if first_token_ms is None:
   166	                first_token_ms = (wall_end - t_start) * 1000
   167
   168	            if emit_timing:
   169	                logger.info(
   170	                    "tts_llm_batch",
   171	                    extra={
   172	                        "speaker": speaker,
   173	                        "request_id": request_id,
   174	                        "batch_index": len(token_batches),
   175	                        "batch_tokens": new_tokens,
   176	                        "llm_batch_wall_ms": round(batch_ms, 3),
   177	                        "llm_batch_gpu_ms": round(batch_gpu_ms, 3) if batch_gpu_ms is not None else None,
   178	                    },
   179	                )
   180	            final_output = request_output
   181
   182	        if final_output is None:
   183	            if emit_timing:
   184	                logger.warning(
   185	                    "tts_llm_no_output",
   186	                    extra={"speaker": speaker, "request_id": request_id},
   187	                )
   188	            print(f"❌ Generation failed: no output")
   189	            if return_timing:
   190	                return None, {"ttfb_ms": 0, "token_total": 0, "batch_count": 0}
   191	            return None
   192
   193	        # Extract generated text (contains BiCodec tokens)
   194	        parse_start = asyncio.get_event_loop().time()
   195	        generated_text = final_output.outputs[0].text
   196	        generated_ids = final_output.outputs[0].token_ids
   197
   198	        # Extract BiCodec tokens using regex
   199	        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)
   200
   201	        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000
   202
   203	        if not semantic_ids or not global_ids:
   204	            if emit_timing:
   205	                logger.error(
   206	                    "tts_bicodec_parse_failed",
   207	                    extra={
   208	                        "speaker": speaker,
   209	                        "request_id": request_id,
   210	                        "semantic_count": len(semantic_ids),
   211	                        "global_count": len(global_ids),
   212	                        "token_total": len(generated_ids),
   213	                    },
   214	                )
   215	            logger.error(
   216	                "❌ No BiCodec tokens found in output",
   217	                extra={
   218	                    "speaker": speaker,
   219	                    "text_length": len(text),
   220	                    "text_preview": text[:100],


nl -ba /home/ubuntu/veenaModal/veena3modal/core/pipeline.py | sed -n '220,420p'


Output:
Chunk ID: 64011a
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 1789
Output:
   220	                    "text_preview": text[:100],
   221	                    "generated_tokens": len(generated_ids),
   222	                    "semantic_tokens": len(semantic_ids),
   223	                    "global_tokens": len(global_ids),
   224	                    "generated_text_preview": generated_text[:500],
   225	                }
   226	            )
   227	            if return_timing:
   228	                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
   229	            return None
   230
   231	        # Validate tokens
   232	        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
   233	            logger.error("❌ BiCodec token validation failed", extra={
   234	                "speaker": speaker,
   235	                "text_length": len(text),
   236	                "semantic_tokens": len(semantic_ids),
   237	                "global_tokens": len(global_ids),
   238	            })
   239	            if return_timing:
   240	                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
   241	            return None
   242
   243	        # Decode to audio
   244	        decode_start = asyncio.get_event_loop().time()
   245	        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
   246	        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000
   247
   248	        if audio_bytes is None:
   249	            if emit_timing:
   250	                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
   251	            logger.error("❌ BiCodec decode failed", extra={
   252	                "speaker": speaker,
   253	                "text_length": len(text),
   254	            })
   255	            if return_timing:
   256	                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
   257	            return None
   258
   259	        total_ms = (asyncio.get_event_loop().time() - t_start) * 1000
   260
   261	        timing_metrics: Dict[str, Any] = {
   262	            "request_id": request_id,
   263	            "speaker": speaker,
   264	            "text_length": len(text),
   265	            "batch_count": len(token_batches),
   266	            "token_total": len(generated_ids),
   267	            "token_batches": token_batches,
   268	            "ttfb_ms": int(first_token_ms or 0),
   269	            "llm_wall_ms": int(sum(batch_wall_ms)),
   270	            "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
   271	            "decode_cpu_ms": int(decode_ms),
   272	            "parse_ms": int(parse_ms),
   273	            "total_ms": int(total_ms),
   274	        }
   275	        if token_batches:
   276	            timing_metrics["tokens_per_batch_min"] = min(token_batches)
   277	            timing_metrics["tokens_per_batch_max"] = max(token_batches)
   278	            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))
   279
   280	        if emit_timing:
   281	            logger.info("tts_llm_complete", extra=timing_metrics)
   282
   283	        from veena3modal.audio.utils import add_wav_header
   284	        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)
   285
   286	        if return_timing:
   287	            return wav_bytes, timing_metrics
   288	        return wav_bytes
   289
   290	    async def generate_speech_indic(
   291	        self,
   292	        speaker: str,
   293	        text: str,
   294	        temperature: float = DEFAULT_TEMPERATURE,
   295	        top_k: int = DEFAULT_TOP_K,
   296	        top_p: float = DEFAULT_TOP_P,
   297	        max_tokens: int = DEFAULT_MAX_TOKENS,
   298	        repetition_penalty: float = 1.05,
   299	        seed: Optional[int] = None,
   300	        return_timing: bool = False,
   301	    ):
   302	        """
   303	        Generate speech audio for Spark TTS (speaker-based).
   304
   305	        This method is kept for backward compatibility with existing API.
   306	        It delegates to generate_speech() with the same implementation.
   307
   308	        Args:
   309	            speaker: Speaker name (one of 12 predefined speakers)
   310	            text: Text to synthesize with inline emotion tags
   311	                Examples:
   312	                - "Hello! Welcome."
   313	                - "Hello <laugh> this is fun!" (will be normalized)
   314	                - "नमस्ते! [excited] आज का दिन बहुत अच्छा है।"
   315	            temperature: Sampling temperature
   316	            top_k: Top-k sampling
   317	            top_p: Nucleus sampling
   318	            max_tokens: Max BiCodec tokens to generate
   319	            seed: Random seed for reproducibility
   320	            return_timing: If true, returns (audio_bytes, metrics_dict)
   321
   322	        Returns:
   323	            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
   324	            If return_timing is true, returns tuple(audio_bytes, metrics).
   325	        """
   326	        # Delegate to generate_speech() with the same implementation for Spark TTS
   327	        return await self.generate_speech(
   328	            speaker=speaker,
   329	            text=text,
   330	            temperature=temperature,
   331	            top_k=top_k,
   332	            top_p=top_p,
   333	            max_tokens=max_tokens,
   334	            repetition_penalty=repetition_penalty,
   335	            seed=seed,
   336	            return_timing=return_timing,
   337	        )
   338
   339	    def _extract_bicodec_tokens(self, generated_text: str) -> Tuple[List[int], List[int]]:
   340	        """
   341	        Extract BiCodec semantic and global tokens from generated text using regex.
   342
   343	        Spark TTS generates tokens in the format:
   344	        - <|bicodec_semantic_{id}|>
   345	        - <|bicodec_global_{id}|>
   346
   347	        Args:
   348	            generated_text: Generated text containing BiCodec token markers
   349
   350	        Returns:
   351	            Tuple of (semantic_ids, global_ids)
   352	            Returns ([], []) if no tokens found
   353	        """
   354	        # Extract semantic tokens
   355	        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", generated_text)
   356	        semantic_ids = [int(t) for t in semantic_matches]
   357
   358	        # Extract global tokens
   359	        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
   360	        global_ids = [int(t) for t in global_matches]
   361
   362	        # Only log if there's an issue
   363	        if not semantic_ids and not global_ids:
   364	            logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")
   365
   366	        return semantic_ids, global_ids


rg -n "initialize_local_runtime|initialize_runtime\(|VEENA3_TIMING_LOGS|generate_speech_streaming\(" /home/ubuntu/veenaModal/scripts -g'*.py'


Output:
Chunk ID: 0254f3
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 21
Output:
/home/ubuntu/veenaModal/scripts/profile_pipeline.py:661:        initialize_runtime(


sed -n '620,760p' /home/ubuntu/veenaModal/scripts/profile_pipeline.py


Output:
Chunk ID: 2eb722
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 946
Output:

    # Token generation rates
    all_rates = []
    for p in profiles:
        all_rates.extend(p.token_generation_rates)
    if all_rates:
        print(f"\n  TOKEN GEN RATE (all samples):")
        print(f"    avg={statistics.mean(all_rates):.0f} tok/s  "
              f"min={min(all_rates):.0f}  max={max(all_rates):.0f}")

# === GPU Memory Snapshot ===

def gpu_snapshot() -> str:
    try:
        import torch
        if not torch.cuda.is_available():
            return "No GPU"
        alloc = torch.cuda.memory_allocated() / 1e9
        reserved = torch.cuda.memory_reserved() / 1e9
        total = torch.cuda.get_device_properties(0).total_memory / 1e9
        return f"GPU: {alloc:.1f}GB alloc, {reserved:.1f}GB reserved / {total:.1f}GB total"
    except Exception:
        return "GPU info unavailable"

# === Main ===

async def main_async(args):
    # Initialize runtime
    logger.info("Initializing TTS runtime for profiling...")
    logger.info(gpu_snapshot())

    from veena3modal.local_server import resolve_model_paths, DEFAULT_LOCAL_MODEL_DIR

    model_path = args.model_path
    llm_path, bicodec_path = resolve_model_paths(model_path)

    from veena3modal.services.tts_runtime import initialize_runtime, get_runtime, is_initialized

    if not is_initialized():
        initialize_runtime(
            model_path=llm_path,
            bicodec_path=bicodec_path,
            device="cuda",
            # gpu_memory_utilization uses VLLM_CONFIG default (0.25 after Tier 1 optimization)
        )

    runtime = get_runtime()
    logger.info(f"Runtime ready: {runtime.model_version}")
    logger.info(gpu_snapshot())

    text = TEXTS.get(args.text, TEXTS["short"])
    concurrency_levels = [int(x) for x in args.concurrency.split(",")]

    # === Single Request Profiles ===
    if not args.concurrency_only:
        logger.info(f"\n{'=' * 80}")
        logger.info(f"SINGLE REQUEST PROFILING (text={args.text}, {len(text)} chars)")
        logger.info(f"{'=' * 80}")

        # Non-streaming
        if not args.stream:
            logger.info("\nProfiling non-streaming...")
            p = await profile_non_streaming(runtime, text, "lipakshi")
            print(p.report())

        # Streaming
        if not args.no_stream:
            logger.info("\nProfiling streaming...")
            p = await profile_streaming(runtime, text, "lipakshi")
            print(p.report())

    # === Concurrency Profiles ===
    for conc in concurrency_levels:
        modes = []
        if not args.stream:
            modes.append(False)
        if not args.no_stream:
            modes.append(True)

        for stream_mode in modes:
            logger.info(f"\nProfiling {conc} concurrent {'streaming' if stream_mode else 'non-streaming'} requests...")
            profiles = await profile_concurrent(runtime, conc, text, stream=stream_mode)
            print_concurrency_summary(profiles, conc, stream_mode)

    logger.info(f"\n{gpu_snapshot()}")
    logger.info("Profiling complete.")

def main():
    parser = argparse.ArgumentParser(description="Veena3 TTS Pipeline Profiler")
    parser.add_argument("--model-path", default=os.path.join(REPO_ROOT, "models", "spark_tts_4speaker"))
    parser.add_argument("--text", default="short", choices=["short", "medium", "long"])
    parser.add_argument("--concurrency", default="1,5,10,20", help="Concurrency levels (comma-separated)")
    parser.add_argument("--stream", action="store_true", help="Stream mode only")
    parser.add_argument("--no-stream", action="store_true", help="Non-stream mode only")
    parser.add_argument("--concurrency-only", action="store_true", help="Skip single-request profiles")
    args = parser.parse_args()
    asyncio.run(main_async(args))

if __name__ == "__main__":
    main()


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/pipeline.py
@@
 import asyncio
 import logging
 import os
+import re
 from statistics import mean, median
 from typing import Optional, List, Dict, Any, Tuple
@@
     async def generate_speech(
         self,
         speaker: str,
         text: str,
         temperature: float = DEFAULT_TEMPERATURE,
@@
         repetition_penalty: float = 1.05,
         seed: Optional[int] = None,
         return_timing: bool = False,
+        request_id: Optional[str] = None,
     ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
@@
         Args:
             speaker: Speaker name (one of 12 predefined speakers)
             text: Text to synthesize (with optional [emotion] tags)
             temperature: Temperature for sampling
@@
             repetition_penalty: repetition penalty
             seed: Random seed for reproducibility
+            request_id: Optional request ID for timing correlation
             return_timing: If true, returns (audio_bytes, metrics_dict)
@@
-        # Use vLLM engine to generate
-        import uuid
-        request_id = f"req-{uuid.uuid4().hex[:12]}"
+        # Use vLLM engine to generate
+        if request_id is None:
+            import uuid
+            request_id = f"req-{uuid.uuid4().hex[:12]}"
@@
         final_output = None
         token_batches: List[int] = []
         batch_wall_ms: List[float] = []
+        batch_gpu_ms: List[float] = []
         total_token_ids = 0
         first_token_ms: Optional[float] = None
         t_start = asyncio.get_event_loop().time()
@@
                 batch_ms = (wall_end - wall_start) * 1000
                 batch_wall_ms.append(batch_ms)

                 if start_event is not None and end_event is not None:
                     end_event.record()
                     torch.cuda.synchronize()
                     batch_gpu_ms = start_event.elapsed_time(end_event)
                 else:
                     batch_gpu_ms = None
@@
                     "batch_tokens": new_tokens,
                     "llm_batch_wall_ms": round(batch_ms, 3),
                     "llm_batch_gpu_ms": round(batch_gpu_ms, 3) if batch_gpu_ms is not None else None,
                 },
                 )
@@
         timing_metrics: Dict[str, Any] = {
             "request_id": request_id,
             "speaker": speaker,
             "text_length": len(text),
             "batch_count": len(token_batches),
             "token_total": len(generated_ids),
             "token_batches": token_batches,
             "ttfb_ms": int(first_token_ms or 0),
             "llm_wall_ms": int(sum(batch_wall_ms)),
             "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
+            "llm_batch_gpu_ms": [int(v) for v in batch_gpu_ms] if batch_gpu_ms else [],
+            "llm_batch_gpu_ms_sum": int(sum(batch_gpu_ms)) if batch_gpu_ms else 0,
             "decode_cpu_ms": int(decode_ms),
             "parse_ms": int(parse_ms),
             "total_ms": int(total_ms),
         }
*** End Patch


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/veenaModal/veena3modal/core/pipeline.py:
        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize (with optional [emotion] tags)
            temperature: Temperature for sampling


sed -n '60,130p' /home/ubuntu/veenaModal/veena3modal/core/pipeline.py


Output:
Chunk ID: b35138
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 653
Output:
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.bicodec_decoder = bicodec_decoder

        print(f"🚀 SparkTTSPipeline initialized")

    async def generate_speech(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
        """
        Generate speech audio (non-streaming) using Spark TTS.

        NOTE: This method signature changed from description-based to speaker-based
        to align with Spark TTS architecture.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize (with optional [emotion] tags)
            temperature: Sampling temperature (default: 0.8 for Spark TTS)
            top_k: Top-k sampling (default: 50)
            top_p: Nucleus sampling (default: 1.0)
            max_tokens: Max BiCodec tokens to generate (default: 2048)
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Build prompt using Spark TTS format
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling for Spark TTS
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
        )

        # Use vLLM engine to generate
        import uuid
        request_id = f"req-{uuid.uuid4().hex[:12]}"

        emit_timing = return_timing or _detailed_timing_enabled()
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Collect all outputs and track generation timing
        final_output = None
        token_batches: List[int] = []
        batch_wall_ms: List[float] = []
        total_token_ids = 0
        first_token_ms: Optional[float] = None


sed -n '130,230p' /home/ubuntu/veenaModal/veena3modal/core/pipeline.py


Output:
Chunk ID: 19a041
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 1012
Output:
        first_token_ms: Optional[float] = None
        t_start = asyncio.get_event_loop().time()

        iterator = results_generator.__aiter__()
        while True:
            wall_start = asyncio.get_event_loop().time()
            start_event = None
            end_event = None
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                start_event = torch.cuda.Event(enable_timing=True)
                end_event = torch.cuda.Event(enable_timing=True)
                start_event.record()

            try:
                request_output = await iterator.__anext__()
            except StopAsyncIteration:
                break

            wall_end = asyncio.get_event_loop().time()
            batch_ms = (wall_end - wall_start) * 1000
            batch_wall_ms.append(batch_ms)

            if start_event is not None and end_event is not None:
                end_event.record()
                torch.cuda.synchronize()
                batch_gpu_ms = start_event.elapsed_time(end_event)
            else:
                batch_gpu_ms = None

            total_generated = len(request_output.outputs[0].token_ids)
            new_tokens = max(0, total_generated - total_token_ids)
            total_token_ids = total_generated
            token_batches.append(new_tokens)

            if first_token_ms is None:
                first_token_ms = (wall_end - t_start) * 1000

            if emit_timing:
                logger.info(
                    "tts_llm_batch",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "batch_index": len(token_batches),
                        "batch_tokens": new_tokens,
                        "llm_batch_wall_ms": round(batch_ms, 3),
                        "llm_batch_gpu_ms": round(batch_gpu_ms, 3) if batch_gpu_ms is not None else None,
                    },
                )
            final_output = request_output

        if final_output is None:
            if emit_timing:
                logger.warning(
                    "tts_llm_no_output",
                    extra={"speaker": speaker, "request_id": request_id},
                )
            print(f"❌ Generation failed: no output")
            if return_timing:
                return None, {"ttfb_ms": 0, "token_total": 0, "batch_count": 0}
            return None

        # Extract generated text (contains BiCodec tokens)
        parse_start = asyncio.get_event_loop().time()
        generated_text = final_output.outputs[0].text
        generated_ids = final_output.outputs[0].token_ids

        # Extract BiCodec tokens using regex
        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)

        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000

        if not semantic_ids or not global_ids:
            if emit_timing:
                logger.error(
                    "tts_bicodec_parse_failed",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "semantic_count": len(semantic_ids),
                        "global_count": len(global_ids),
                        "token_total": len(generated_ids),
                    },
                )
            logger.error(
                "❌ No BiCodec tokens found in output",
                extra={
                    "speaker": speaker,
                    "text_length": len(text),
                    "text_preview": text[:100],
                    "generated_tokens": len(generated_ids),
                    "semantic_tokens": len(semantic_ids),
                    "global_tokens": len(global_ids),
                    "generated_text_preview": generated_text[:500],
                }
            )
            if return_timing:
                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
            return None


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/pipeline.py
@@
 import asyncio
 import logging
 import os
+import re
 from statistics import mean, median
 from typing import Optional, List, Dict, Any, Tuple
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/pipeline.py
@@
     async def generate_speech(
         self,
         speaker: str,
         text: str,
         temperature: float = DEFAULT_TEMPERATURE,
         top_k: int = DEFAULT_TOP_K,
         top_p: float = DEFAULT_TOP_P,
         max_tokens: int = DEFAULT_MAX_TOKENS,
         repetition_penalty: float = 1.05,
         seed: Optional[int] = None,
         return_timing: bool = False,
+        request_id: Optional[str] = None,
     ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/pipeline.py
@@
-        # Use vLLM engine to generate
-        import uuid
-        request_id = f"req-{uuid.uuid4().hex[:12]}"
+        # Use vLLM engine to generate
+        if request_id is None:
+            import uuid
+            request_id = f"req-{uuid.uuid4().hex[:12]}"
@@
         # Collect all outputs and track generation timing
         final_output = None
         token_batches: List[int] = []
         batch_wall_ms: List[float] = []
+        batch_gpu_ms: List[float] = []
         total_token_ids = 0
         first_token_ms: Optional[float] = None
         t_start = asyncio.get_event_loop().time()
@@
-            if start_event is not None and end_event is not None:
-                end_event.record()
-                torch.cuda.synchronize()
-                batch_gpu_ms = start_event.elapsed_time(end_event)
-            else:
-                batch_gpu_ms = None
+            batch_gpu_ms_value: Optional[float] = None
+            if start_event is not None and end_event is not None:
+                end_event.record()
+                torch.cuda.synchronize()
+                batch_gpu_ms_value = start_event.elapsed_time(end_event)
+            if batch_gpu_ms_value is not None:
+                batch_gpu_ms.append(batch_gpu_ms_value)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/pipeline.py
@@
                     extra={
                         "speaker": speaker,
                         "request_id": request_id,
                         "batch_index": len(token_batches),
                         "batch_tokens": new_tokens,
                         "llm_batch_wall_ms": round(batch_ms, 3),
-                        "llm_batch_gpu_ms": round(batch_gpu_ms, 3) if batch_gpu_ms is not None else None,
+                        "llm_batch_gpu_ms": round(batch_gpu_ms_value, 3)
+                        if batch_gpu_ms_value is not None
+                        else None,
                     },
                 )
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/pipeline.py
@@
             print(f"❌ Generation failed: no output")
             if return_timing:
-                return None, {"ttfb_ms": 0, "token_total": 0, "batch_count": 0}
+                return None, {
+                    "request_id": request_id,
+                    "ttfb_ms": 0,
+                    "token_total": 0,
+                    "batch_count": 0,
+                }
             return None
@@
             if return_timing:
-                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
+                return None, {
+                    "request_id": request_id,
+                    "ttfb_ms": int(first_token_ms or 0),
+                    "token_total": len(generated_ids),
+                    "batch_count": len(token_batches),
+                }
             return None
@@
             if return_timing:
-                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
+                return None, {
+                    "request_id": request_id,
+                    "ttfb_ms": int(first_token_ms or 0),
+                    "token_total": len(generated_ids),
+                    "batch_count": len(token_batches),
+                }
             return None
@@
             if return_timing:
-                return None, {"ttfb_ms": int(first_token_ms or 0), "token_total": len(generated_ids), "batch_count": len(token_batches)}
+                return None, {
+                    "request_id": request_id,
+                    "ttfb_ms": int(first_token_ms or 0),
+                    "token_total": len(generated_ids),
+                    "batch_count": len(token_batches),
+                }
             return None
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


sed -n '250,320p' /home/ubuntu/veenaModal/veena3modal/core/pipeline.py


Output:
Chunk ID: c063b3
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 703
Output:
            logger.error("❌ BiCodec token validation failed", extra={
                "speaker": speaker,
                "text_length": len(text),
                "semantic_tokens": len(semantic_ids),
                "global_tokens": len(global_ids),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Decode to audio
        decode_start = asyncio.get_event_loop().time()
        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000

        if audio_bytes is None:
            if emit_timing:
                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
            logger.error("❌ BiCodec decode failed", extra={
                "speaker": speaker,
                "text_length": len(text),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        total_ms = (asyncio.get_event_loop().time() - t_start) * 1000

        timing_metrics: Dict[str, Any] = {
            "request_id": request_id,
            "speaker": speaker,
            "text_length": len(text),
            "batch_count": len(token_batches),
            "token_total": len(generated_ids),
            "token_batches": token_batches,
            "ttfb_ms": int(first_token_ms or 0),
            "llm_wall_ms": int(sum(batch_wall_ms)),
            "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
            "decode_cpu_ms": int(decode_ms),
            "parse_ms": int(parse_ms),
            "total_ms": int(total_ms),
        }
        if token_batches:
            timing_metrics["tokens_per_batch_min"] = min(token_batches)
            timing_metrics["tokens_per_batch_max"] = max(token_batches)
            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))

        if emit_timing:
            logger.info("tts_llm_complete", extra=timing_metrics)

        from veena3modal.audio.utils import add_wav_header
        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)

        if return_timing:
            return wav_bytes, timing_metrics
        return wav_bytes

    async def generate_speech_indic(
        self,
        speaker: str,
        text: str,


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/pipeline.py
@@
         timing_metrics: Dict[str, Any] = {
             "request_id": request_id,
             "speaker": speaker,
             "text_length": len(text),
             "batch_count": len(token_batches),
             "token_total": len(generated_ids),
             "token_batches": token_batches,
             "ttfb_ms": int(first_token_ms or 0),
             "llm_wall_ms": int(sum(batch_wall_ms)),
             "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
+            "llm_gpu_batch_ms_total": int(sum(batch_gpu_ms)),
+            "llm_gpu_batch_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
+            "llm_gpu_batch_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
+            "llm_gpu_batch_ms_p50": int(median(batch_gpu_ms)) if batch_gpu_ms else 0,
             "decode_cpu_ms": int(decode_ms),
             "parse_ms": int(parse_ms),
             "total_ms": int(total_ms),
         }
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/core/pipeline.py
@@
     async def generate_speech_indic(
         self,
         speaker: str,
         text: str,
         temperature: float = DEFAULT_TEMPERATURE,
         top_k: int = DEFAULT_TOP_K,
         top_p: float = DEFAULT_TOP_P,
         max_tokens: int = DEFAULT_MAX_TOKENS,
         repetition_penalty: float = 1.05,
         seed: Optional[int] = None,
         return_timing: bool = False,
+        request_id: Optional[str] = None,
     ):
@@
             repetition_penalty=repetition_penalty,
             seed=seed,
             return_timing=return_timing,
+            request_id=request_id,
         )
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py
@@
 async def generate_speech(
     text: str,
     speaker: str,
     temperature: float = 0.8,
     top_k: int = 50,
@@
     repetition_penalty: float = 1.05,
     seed: Optional[int] = None,
     output_sample_rate: str = "16khz",
+    request_id: Optional[str] = None,
 ) -> Tuple[Optional[bytes], Dict[str, Any]]:
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py
@@
     runtime = get_runtime()
     metrics = {
+        "request_id": request_id,
         "ttfb_ms": 0,
         "generation_ms": 0,
         "tokens_generated": 0,
         "audio_duration_seconds": 0.0,
@@
         "sr_applied": False,
         "output_sample_rate": 16000,
     }
+
+    if request_id is None:
+        import uuid
+        request_id = f"runtime-ns-{uuid.uuid4().hex[:10]}"
+        metrics["request_id"] = request_id
@@
                 "speaker": speaker,
                 "text_length": len(text),
                 "request_inflight": active_requests,
+                "request_id": request_id,
                 "temperature": temperature,
                 "top_k": top_k,
                 "top_p": top_p,
                 "max_tokens": max_tokens,
                 "return_timing": _timing_enabled(),
@@
             audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
                 **gen_kwargs,
                 return_timing=True,
+                request_id=request_id,
             )
         else:
-            audio_bytes = await runtime.pipeline.generate_speech_indic(**gen_kwargs)
+            audio_bytes = await runtime.pipeline.generate_speech_indic(
+                **gen_kwargs,
+                request_id=request_id,
+            )
@@
                 metrics.update({
                     "llm_token_total": pipeline_timings.get("token_total", 0),
                     "llm_batch_count": pipeline_timings.get("batch_count", 0),
+                    "llm_batch_gpu_ms_total": pipeline_timings.get("llm_gpu_batch_ms_total", 0),
+                    "llm_batch_gpu_ms_min": pipeline_timings.get("llm_gpu_batch_ms_min", 0),
+                    "llm_batch_gpu_ms_max": pipeline_timings.get("llm_gpu_batch_ms_max", 0),
+                    "llm_batch_gpu_ms_p50": pipeline_timings.get("llm_batch_gpu_ms_p50", 0),
                     "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
                     "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
                     "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
                     "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
                     "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
                     "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
                     "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
                 })
*** End Patch


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py:
                "speaker": speaker,
                "text_length": len(text),
                "request_inflight": active_requests,
                "temperature": temperature,
                "top_k": top_k,
                "top_p": top_p,
                "max_tokens": max_tokens,
                "return_timing": _timing_enabled(),


I’m re-running smaller, exact-match patches now because one hunk missed due local differences in the existing logger payload.


sed -n '392,470p' /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py


Output:
Chunk ID: a52cad
Wall time: 0.0537 seconds
Process exited with code 0
Original token count: 735
Output:
        Tuple of (audio_bytes, metrics_dict)
        audio_bytes: WAV audio bytes (16kHz or 48kHz, 16-bit PCM) or None if failed
        metrics_dict: Dictionary with timing metrics

    Raises:
        RuntimeError: If runtime not initialized
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()
    metrics = {
        "ttfb_ms": 0,
        "generation_ms": 0,
        "tokens_generated": 0,
        "audio_duration_seconds": 0.0,
        "sr_applied": False,
        "output_sample_rate": 16000,
    }

    active_requests = _with_inflight_delta(1)
    start_time = time.time()

    pipeline_timings = {}

    try:
        logger.info(
            "tts_runtime_nonstream_start",
            extra={
                "speaker": speaker,
                "text_length": len(text),
                "request_inflight": active_requests,
                "temperature": temperature,
                "top_k": top_k,
                "top_p": top_p,
                "max_tokens": max_tokens,
            },
        )

        gen_kwargs = dict(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
        )

        if _timing_enabled():
            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
                **gen_kwargs,
                return_timing=True,
            )
        else:
            audio_bytes = await runtime.pipeline.generate_speech_indic(**gen_kwargs)

        generation_time = time.time() - start_time
        metrics["generation_ms"] = int(generation_time * 1000)
        metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time

        if pipeline_timings:
            metrics.update({
                "llm_token_total": pipeline_timings.get("token_total", 0),
                "llm_batch_count": pipeline_timings.get("batch_count", 0),
                "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
                "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
                "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
                "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
                "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
                "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
            })

        if audio_bytes:
            # Apply super-resolution if requested and available
            sample_rate = 16000
            logger.info(f"SR check: output_sample_rate={output_sample_rate!r}, "


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py
@@
     runtime = get_runtime()
     metrics = {
+        "request_id": request_id,
         "ttfb_ms": 0,
         "generation_ms": 0,
         "tokens_generated": 0,
         "audio_duration_seconds": 0.0,
         "sr_applied": False,
         "output_sample_rate": 16000,
     }
+
+    if request_id is None:
+        import uuid
+        request_id = f"runtime-ns-{uuid.uuid4().hex[:10]}"
+        metrics["request_id"] = request_id
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py
@@
             extra={
                 "speaker": speaker,
+                "request_id": request_id,
                 "text_length": len(text),
                 "request_inflight": active_requests,
                 "temperature": temperature,
                 "top_k": top_k,
                 "top_p": top_p,
@@
         if _timing_enabled():
             audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
                 **gen_kwargs,
                 return_timing=True,
+                request_id=request_id,
             )
         else:
-            audio_bytes = await runtime.pipeline.generate_speech_indic(**gen_kwargs)
+            audio_bytes = await runtime.pipeline.generate_speech_indic(
+                **gen_kwargs,
+                request_id=request_id,
+            )
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py
@@
             metrics.update({
                 "llm_token_total": pipeline_timings.get("token_total", 0),
                 "llm_batch_count": pipeline_timings.get("batch_count", 0),
+                "llm_batch_gpu_ms_total": pipeline_timings.get("llm_gpu_batch_ms_total", 0),
+                "llm_batch_gpu_ms_min": pipeline_timings.get("llm_gpu_batch_ms_min", 0),
+                "llm_batch_gpu_ms_max": pipeline_timings.get("llm_gpu_batch_ms_max", 0),
+                "llm_batch_gpu_ms_p50": pipeline_timings.get("llm_gpu_batch_ms_p50", 0),
                 "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
                 "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
                 "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
                 "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
                 "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py
@@
         finally:
             active_requests = _with_inflight_delta(-1)
             if _timing_enabled():
                 logger.info(
                     "tts_runtime_nonstream_inflight",
                     extra={"active_requests": active_requests},
                 )
*** End Patch


Output: {"output":"Success. Updated the following files:\nM /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


sed -n '418,512p' /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py