Output:
Chunk ID: d5a86d
Wall time: 0.0516 seconds
Process exited with code 0
Original token count: 3616
Output:
   560	        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
   561
   562	        # Check if chunking is needed
   563	        if long_processor.should_chunk(text):
   564	            metrics["text_chunked"] = True
   565	            audio_bytes = await long_processor.generate_with_chunking(
   566	                text=text,
   567	                speaker=speaker,
   568	                temperature=temperature,
   569	                top_k=top_k,
   570	                top_p=top_p,
   571	                max_tokens=max_tokens,
   572	                repetition_penalty=repetition_penalty,
   573	                seed=seed,
   574	                sample_rate=sample_rate,
   575	            )
   576	        else:
   577	            # Short text: direct generation
   578	            audio_bytes = await runtime.pipeline.generate_speech_indic(
   579	                speaker=speaker,
   580	                text=text,
   581	                temperature=temperature,
   582	                top_k=top_k,
   583	                top_p=top_p,
   584	                max_tokens=max_tokens,
   585	                repetition_penalty=repetition_penalty,
   586	                seed=seed,
   587	            )
   588
   589	        generation_time = time.time() - start_time
   590	        metrics["generation_ms"] = int(generation_time * 1000)
   591	        metrics["ttfb_ms"] = metrics["generation_ms"]
   592
   593	        if audio_bytes:
   594	            # Apply super-resolution if requested and available
   595	            final_sample_rate = 16000
   596	            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
   597	                try:
   598	                    sr_start = time.time()
   599	                    logger.info(f"Applying super-resolution to chunked audio ({len(audio_bytes)} bytes)...")
   600	                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
   601	                    sr_time = (time.time() - sr_start) * 1000
   602	                    metrics["sr_ms"] = int(sr_time)
   603	                    metrics["sr_applied"] = True
   604	                    final_sample_rate = 48000
   605	                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
   606	                except Exception as e:
   607	                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
   608	                    import traceback
   609	                    traceback.print_exc()
   610
   611	            metrics["output_sample_rate"] = final_sample_rate
   612	            audio_duration = (len(audio_bytes) - 44) / (final_sample_rate * 2)
   613	            metrics["audio_duration_seconds"] = max(0.0, audio_duration)
   614
   615	        return audio_bytes, metrics
   616
   617	    except Exception as e:
   618	        logger.error(f"Chunked speech generation failed: {e}")
   619	        raise
   620
   621
   622	from typing import AsyncGenerator
   623
   624
   625	async def generate_speech_streaming(
   626	    text: str,
   627	    speaker: str,
   628	    temperature: float = 0.8,
   629	    top_k: int = 50,
   630	    top_p: float = 1.0,
   631	    max_tokens: int = 4096,
   632	    repetition_penalty: float = 1.05,
   633	    seed: Optional[int] = None,
   634	    enable_chunking: bool = True,
   635	) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
   636	    """
   637	    Generate speech audio with true streaming (yields chunks as they're generated).
   638
   639	    This is the core streaming implementation for M4.
   640	    First yield includes WAV header + first PCM chunk.
   641	    Subsequent yields are raw PCM chunks.
   642
   643	    Args:
   644	        text: Text to synthesize (already normalized)
   645	        speaker: Internal speaker name (resolved)
   646	        temperature: Sampling temperature
   647	        top_k: Top-k sampling
   648	        top_p: Nucleus sampling
   649	        max_tokens: Maximum tokens to generate
   650	        repetition_penalty: Repetition penalty
   651	        seed: Random seed for reproducibility
   652	        enable_chunking: Enable text chunking for long inputs with voice consistency
   653
   654	    Yields:
   655	        Tuple of (audio_bytes, metrics_dict)
   656	        - First yield: WAV header (44 bytes) prepended to first PCM chunk
   657	        - Subsequent yields: Raw PCM chunks (int16, 16kHz)
   658	        - Final yield: metrics_dict has final timing info
   659
   660	    Raises:
   661	        RuntimeError: If runtime not initialized or streaming pipeline unavailable
   662	    """
   663	    if not is_initialized():
   664	        raise RuntimeError("TTS runtime not initialized")
   665
   666	    runtime = get_runtime()
   667
   668	    if runtime.streaming_pipeline is None:
   669	        raise RuntimeError("Streaming pipeline not available")
   670
   671	    # Import audio utils for WAV header
   672	    from veena3modal.audio.utils import create_wav_header
   673
   674	    # Metrics tracking
   675	    metrics = {
   676	        "ttfb_ms": 0,
   677	        "chunks_sent": 0,
   678	        "total_bytes": 0,
   679	        "audio_duration_seconds": 0.0,
   680	        "text_chunked": False,
   681	    }
   682
   683	    start_time = time.time()
   684	    first_chunk_time = None
   685	    total_pcm_bytes = 0
   686	    sample_rate = 16000  # BiCodec sample rate
   687
   688	    # Check if we need text chunking
   689	    from veena3modal.processing.long_text_processor import LongTextProcessor
   690	    long_processor = LongTextProcessor(pipeline=runtime.pipeline)
   691	    needs_chunking = enable_chunking and long_processor.should_chunk(text)
   692
   693	    if needs_chunking:
   694	        # Chunked streaming with voice consistency (global token caching)
   695	        metrics["text_chunked"] = True
   696	        async for audio_chunk, chunk_metrics in _stream_chunked_text(
   697	            runtime, long_processor, text, speaker, temperature, top_k, top_p,
   698	            max_tokens, repetition_penalty, seed, sample_rate
   699	        ):
   700	            # First chunk: prepend WAV header
   701	            if first_chunk_time is None:
   702	                first_chunk_time = time.time()
   703	                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
   704	                # Create streaming WAV header (size=0 for unknown length)
   705	                wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
   706	                audio_chunk = wav_header + audio_chunk
   707
   708	            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
   709	            metrics["chunks_sent"] += 1
   710	            metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
   711	            metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
   712
   713	            yield audio_chunk, metrics
   714	    else:
   715	        # Simple streaming (no text chunking)
   716	        # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
   717	        # use continuation mode to skip the ~110ms global token pre-roll.
   718	        # The model jumps straight to semantic token generation.
   719	        cached_globals = runtime.speaker_global_cache.get(speaker)
   720
   721	        if cached_globals:
   722	            # Fast path: use cached globals, skip global token generation entirely
   723	            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
   724	                speaker=speaker,
   725	                text=text,
   726	                global_ids=cached_globals,
   727	                temperature=temperature,
   728	                top_k=top_k,
   729	                top_p=top_p,
   730	                max_tokens=max_tokens,
   731	                repetition_penalty=repetition_penalty,
   732	                seed=seed,
   733	            )
   734	        else:
   735	            # Fallback: no cached globals, generate them inline (original path)
   736	            stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
   737	                speaker=speaker,
   738	                text=text,
   739	                temperature=temperature,
   740	                top_k=top_k,
   741	                top_p=top_p,
   742	                max_tokens=max_tokens,
   743	                repetition_penalty=repetition_penalty,
   744	                seed=seed,
   745	            )
   746
   747	        async for audio_chunk in stream_gen:
   748	            # First chunk: prepend WAV header
   749	            if first_chunk_time is None:
   750	                first_chunk_time = time.time()
   751	                metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
   752	                wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
   753	                audio_chunk = wav_header + audio_chunk
   754
   755	            total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
   756	            metrics["chunks_sent"] += 1
   757	            metrics["total_bytes"] = total_pcm_bytes + 44
   758	            metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
   759
   760	            yield audio_chunk, metrics
   761
   762
   763	async def _stream_chunked_text(
   764	    runtime: TTSRuntime,
   765	    long_processor,
   766	    text: str,
   767	    speaker: str,
   768	    temperature: float,
   769	    top_k: int,
   770	    top_p: float,
   771	    max_tokens: int,
   772	    repetition_penalty: float,
   773	    seed: Optional[int],
   774	    sample_rate: int,
   775	) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
   776	    """
   777	    Internal helper: stream chunked text with voice consistency.
   778
   779	    Uses global token caching from first chunk to maintain voice across chunks.
   780	    """
   781	    from veena3modal.audio.crossfade import crossfade_bytes_int16
   782
   783	    # Chunk text using the long text processor's chunker
   784	    chunks = long_processor.chunk_text(text)
   785
   786	    if not chunks:
   787	        return
   788
   789	    captured_globals: Optional[List[int]] = None
   790	    previous_chunk_tail: Optional[bytes] = None
   791	    chunk_metrics = {"chunks_processed": 0}
   792
   793	    for i, chunk_text in enumerate(chunks):
   794	        chunk_metrics["chunks_processed"] = i + 1
   795
   796	        if i == 0:
   797	            # First chunk: capture global tokens for voice consistency
   798	            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
   799	                speaker=speaker,
   800	                text=chunk_text,
   801	                temperature=temperature,
   802	                top_k=top_k,
   803	                top_p=top_p,
   804	                max_tokens=max_tokens,
   805	                repetition_penalty=repetition_penalty,
   806	                seed=seed,
   807	            ):
   808	                if captured_globals is None and global_ids:
   809	                    captured_globals = global_ids
   810
   811	                # Crossfade with previous tail (inter-chunk stitching)
   812	                to_emit, previous_chunk_tail = crossfade_bytes_int16(
   813	                    previous_chunk_tail,
   814	                    audio_bytes,
   815	                    sample_rate_hz=sample_rate,
   816	                    crossfade_ms=50,
   817	                )
   818
   819	                if to_emit:
   820	                    yield to_emit, chunk_metrics
   821	        else:
   822	            # Continuation chunks: use captured globals for voice consistency
   823	            if captured_globals is None:
   824	                logger.warning(f"No captured globals for chunk {i+1}, using regular streaming")
   825	                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
   826	                    speaker=speaker,
   827	                    text=chunk_text,
   828	                    temperature=temperature,
   829	                    top_k=top_k,
   830	                    top_p=top_p,
   831	                    max_tokens=max_tokens,
   832	                    repetition_penalty=repetition_penalty,
   833	                    seed=seed,
   834	                ):
   835	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   836	                        previous_chunk_tail,
   837	                        audio_bytes,
   838	                        sample_rate_hz=sample_rate,
   839	                        crossfade_ms=50,
   840	                    )
   841	                    if to_emit:
   842	                        yield to_emit, chunk_metrics
   843	            else:
   844	                # Use continuation method with cached globals
   845	                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
   846	                    speaker=speaker,
   847	                    text=chunk_text,
   848	                    global_ids=captured_globals,
   849	                    temperature=temperature,
   850	                    top_k=top_k,
   851	                    top_p=top_p,
   852	                    max_tokens=max_tokens,
   853	                    repetition_penalty=repetition_penalty,
   854	                    seed=seed,
   855	                ):
   856	                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
   857	                        previous_chunk_tail,
   858	                        audio_bytes,
   859	                        sample_rate_hz=sample_rate,
   860	                        crossfade_ms=50,
   861	                    )
   862	                    if to_emit:
   863	                        yield to_emit, chunk_metrics
   864
   865	    # Flush remaining tail
   866	    if previous_chunk_tail:
   867	        yield previous_chunk_tail, chunk_metrics


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Planning pipeline file inspection**


nl -ba veena3modal/core/pipeline.py | sed -n '1,360p'


Output:
Chunk ID: 9e70d3
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 2521
Output:
     1	"""
     2	Spark TTS Generation Pipeline
     3
     4	End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
     5	Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
     6	"""
     7
     8	import asyncio
     9	import logging
    10	import re
    11	from typing import Optional, List, Dict, Any, Tuple
    12	from vllm import SamplingParams
    13	import torch
    14
    15	logger = logging.getLogger(__name__)
    16
    17	from veena3modal.core.constants import (
    18	    TRAINING_STOP_TOKEN_IDS,
    19	    DEFAULT_TEMPERATURE,
    20	    DEFAULT_TOP_K,
    21	    DEFAULT_TOP_P,
    22	    DEFAULT_MAX_TOKENS,
    23	    DEFAULT_SEED,
    24	    AUDIO_SAMPLE_RATE,
    25	)
    26
    27
    28	class SparkTTSPipeline:
    29	    """
    30	    End-to-end TTS pipeline for Spark TTS with BiCodec.
    31
    32	    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    33	    """
    34
    35	    def __init__(
    36	        self,
    37	        model,
    38	        prompt_builder,
    39	        bicodec_decoder,
    40	    ):
    41	        """
    42	        Initialize pipeline.
    43
    44	        Args:
    45	            model: SparkTTS Model instance
    46	            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
    47	            bicodec_decoder: BiCodecDecoder instance
    48	        """
    49	        self.model = model
    50	        self.prompt_builder = prompt_builder
    51	        self.bicodec_decoder = bicodec_decoder
    52
    53	        print(f"🚀 SparkTTSPipeline initialized")
    54
    55	    async def generate_speech(
    56	        self,
    57	        speaker: str,
    58	        text: str,
    59	        temperature: float = DEFAULT_TEMPERATURE,
    60	        top_k: int = DEFAULT_TOP_K,
    61	        top_p: float = DEFAULT_TOP_P,
    62	        max_tokens: int = DEFAULT_MAX_TOKENS,
    63	        repetition_penalty: float = 1.05,
    64	        seed: Optional[int] = None,
    65	    ) -> Optional[bytes]:
    66	        """
    67	        Generate speech audio (non-streaming) using Spark TTS.
    68
    69	        NOTE: This method signature changed from description-based to speaker-based
    70	        to align with Spark TTS architecture.
    71
    72	        Args:
    73	            speaker: Speaker name (one of 12 predefined speakers)
    74	            text: Text to synthesize (with optional [emotion] tags)
    75	            temperature: Sampling temperature (default: 0.8 for Spark TTS)
    76	            top_k: Top-k sampling (default: 50)
    77	            top_p: Nucleus sampling (default: 1.0)
    78	            max_tokens: Max BiCodec tokens to generate (default: 2048)
    79	            seed: Random seed for reproducibility
    80
    81	        Returns:
    82	            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed
    83	        """
    84	        # Build prompt using Spark TTS format
    85	        prompt = self.prompt_builder.build_prefix(speaker, text)
    86
    87	        # Configure sampling for Spark TTS
    88	        sampling_params = SamplingParams(
    89	            temperature=temperature,
    90	            top_k=top_k,
    91	            top_p=top_p,
    92	            max_tokens=max_tokens,
    93	            repetition_penalty=repetition_penalty,  # Prevent token repetition (e.g., token 484 loop)
    94	            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>"
    95	            skip_special_tokens=False,  # Keep special tokens for BiCodec extraction
    96	            seed=seed,
    97	        )
    98
    99	        # Generate tokens
   100	        import uuid
   101	        request_id = f"req-{uuid.uuid4().hex[:12]}"
   102
   103	        # Use vLLM engine to generate
   104	        results_generator = self.model.engine.generate(
   105	            prompt=prompt,
   106	            sampling_params=sampling_params,
   107	            request_id=request_id,
   108	        )
   109
   110	        # Collect all outputs
   111	        final_output = None
   112	        async for request_output in results_generator:
   113	            final_output = request_output
   114
   115	        if final_output is None:
   116	            print(f"❌ Generation failed: no output")
   117	            return None
   118
   119	        # Extract generated text (contains BiCodec tokens)
   120	        generated_text = final_output.outputs[0].text
   121	        generated_ids = final_output.outputs[0].token_ids
   122
   123	        # Extract BiCodec tokens using regex
   124	        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)
   125
   126	        if not semantic_ids or not global_ids:
   127	            logger.error(
   128	                "❌ No BiCodec tokens found in output",
   129	                extra={
   130	                    "speaker": speaker,
   131	                    "text_length": len(text),
   132	                    "text_preview": text[:100],
   133	                    "generated_tokens": len(generated_ids),
   134	                    "semantic_tokens": len(semantic_ids),
   135	                    "global_tokens": len(global_ids),
   136	                    "generated_text_preview": generated_text[:500],
   137	                }
   138	            )
   139	            return None
   140
   141	        # Validate tokens
   142	        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
   143	            logger.error("❌ BiCodec token validation failed", extra={
   144	                "speaker": speaker,
   145	                "text_length": len(text),
   146	                "semantic_tokens": len(semantic_ids),
   147	                "global_tokens": len(global_ids),
   148	            })
   149	            return None
   150
   151	        # Decode to audio
   152	        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
   153
   154	        if audio_bytes is None:
   155	            logger.error("❌ BiCodec decode failed", extra={
   156	                "speaker": speaker,
   157	                "text_length": len(text),
   158	            })
   159	            return None
   160
   161	        # Add WAV header
   162	        from veena3modal.audio.utils import add_wav_header
   163	        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)
   164
   165	        return wav_bytes
   166
   167	    async def generate_speech_indic(
   168	        self,
   169	        speaker: str,
   170	        text: str,
   171	        temperature: float = DEFAULT_TEMPERATURE,
   172	        top_k: int = DEFAULT_TOP_K,
   173	        top_p: float = DEFAULT_TOP_P,
   174	        max_tokens: int = DEFAULT_MAX_TOKENS,
   175	        repetition_penalty: float = 1.05,
   176	        seed: Optional[int] = None,
   177	    ) -> Optional[bytes]:
   178	        """
   179	        Generate speech audio for Spark TTS (speaker-based).
   180
   181	        This method is kept for backward compatibility with existing API.
   182	        It delegates to generate_speech() with the same implementation.
   183
   184	        Args:
   185	            speaker: Speaker name (one of 12 predefined speakers)
   186	            text: Text to synthesize with inline emotion tags
   187	                Examples:
   188	                - "Hello! Welcome."
   189	                - "[laughs] Hello there!"
   190	                - "Hello <laugh> this is fun!" (will be normalized)
   191	                - "नमस्ते! [excited] आज का दिन बहुत अच्छा है।"
   192	            temperature: Sampling temperature
   193	            top_k: Top-k sampling
   194	            top_p: Nucleus sampling
   195	            max_tokens: Max BiCodec tokens to generate
   196	            seed: Random seed for reproducibility
   197
   198	        Returns:
   199	            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed
   200	        """
   201	        # Delegate to generate_speech (same implementation for Spark TTS)
   202	        return await self.generate_speech(
   203	            speaker=speaker,
   204	            text=text,
   205	            temperature=temperature,
   206	            top_k=top_k,
   207	            top_p=top_p,
   208	            max_tokens=max_tokens,
   209	            repetition_penalty=repetition_penalty,
   210	            seed=seed,
   211	        )
   212
   213	    def _extract_bicodec_tokens(self, generated_text: str) -> Tuple[List[int], List[int]]:
   214	        """
   215	        Extract BiCodec semantic and global tokens from generated text using regex.
   216
   217	        Spark TTS generates tokens in the format:
   218	        - <|bicodec_semantic_{id}|>
   219	        - <|bicodec_global_{id}|>
   220
   221	        Args:
   222	            generated_text: Generated text containing BiCodec token markers
   223
   224	        Returns:
   225	            Tuple of (semantic_ids, global_ids)
   226	            Returns ([], []) if no tokens found
   227	        """
   228	        # Extract semantic tokens
   229	        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", generated_text)
   230	        semantic_ids = [int(t) for t in semantic_matches]
   231
   232	        # Extract global tokens
   233	        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
   234	        global_ids = [int(t) for t in global_matches]
   235
   236	        # Only log if there's an issue
   237	        if not semantic_ids and not global_ids:
   238	            logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")
   239
   240	        return semantic_ids, global_ids
   241


Summary: auto


**Designing detailed LLM and decode profiling**


**Refining metric naming and extraction approach**


I’ve pinpointed the instrumentation insertion point: `SparkTTSPipeline` is the shared non-streaming path used by the optimized local server. I’m now adding a profiled generation method there (LLM batch timings, token rates, vLLM internal timing extraction, BiCodec GPU/CPU split) and then wiring those metrics into API headers.


Input:
*** Begin Patch
*** Update File: veena3modal/core/pipeline.py
@@
-import asyncio
 import logging
 import re
+import time
+import uuid
 from typing import Optional, List, Dict, Any, Tuple
 from vllm import SamplingParams
 import torch
@@
 class SparkTTSPipeline:
@@
         self.bicodec_decoder = bicodec_decoder

         print(f"🚀 SparkTTSPipeline initialized")
+
+    @staticmethod
+    def _p50(values: List[float]) -> float:
+        """Compute a stable p50 for small lists."""
+        if not values:
+            return 0.0
+        ordered = sorted(values)
+        return ordered[(len(ordered) - 1) // 2]
+
+    @staticmethod
+    def _to_number(value: Any) -> Optional[float]:
+        """Best-effort conversion of scalar values to float."""
+        if value is None:
+            return None
+        if isinstance(value, (int, float)):
+            return float(value)
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return None
+
+    def _extract_request_metrics(self, raw_metrics: Any) -> Dict[str, float]:
+        """
+        Convert vLLM request_output.metrics into a numeric dictionary.
+
+        vLLM may return a dataclass-like object or a plain dict depending on version.
+        """
+        if raw_metrics is None:
+            return {}
+
+        numeric: Dict[str, float] = {}
+        if isinstance(raw_metrics, dict):
+            items = raw_metrics.items()
+        else:
+            names = (
+                "arrival_time",
+                "first_scheduled_time",
+                "first_token_time",
+                "finished_time",
+                "time_in_queue",
+                "scheduler_time",
+                "model_forward_time",
+                "model_execute_time",
+            )
+            items = []
+            for name in names:
+                if hasattr(raw_metrics, name):
+                    items.append((name, getattr(raw_metrics, name)))
+
+        for key, value in items:
+            parsed = self._to_number(value)
+            if parsed is not None:
+                numeric[str(key)] = parsed
+        return numeric
+
+    def _apply_stats(self, perf: Dict[str, Any], prefix: str, values: List[float]) -> None:
+        """Populate total/min/max/p50 stats for a list of ms samples."""
+        if not values:
+            return
+        perf[f"{prefix}_total"] = float(sum(values))
+        perf[f"{prefix}_min"] = float(min(values))
+        perf[f"{prefix}_max"] = float(max(values))
+        perf[f"{prefix}_p50"] = float(self._p50(values))
+
+    async def generate_speech_profiled(
+        self,
+        speaker: str,
+        text: str,
+        temperature: float = DEFAULT_TEMPERATURE,
+        top_k: int = DEFAULT_TOP_K,
+        top_p: float = DEFAULT_TOP_P,
+        max_tokens: int = DEFAULT_MAX_TOKENS,
+        repetition_penalty: float = 1.05,
+        seed: Optional[int] = None,
+    ) -> Tuple[Optional[bytes], Dict[str, Any]]:
+        """
+        Generate speech with detailed per-request timing metrics.
+
+        Returns:
+            (wav_bytes, perf_dict)
+        """
+        from veena3modal.audio.utils import add_wav_header
+
+        perf: Dict[str, Any] = {
+            "llm_prompt_token_total": 0,
+            "llm_token_total": 0,
+            "semantic_token_total": 0,
+            "global_token_total": 0,
+            "llm_batch_count": 0,
+            "llm_parse_ms": 0.0,
+            "llm_decode_calls": 0,
+            "bicodec_decode_wall_ms": 0.0,
+            "bicodec_decode_gpu_ms": 0.0,
+            "bicodec_decode_cpu_ms": 0.0,
+            "wav_pack_ms": 0.0,
+        }
+
+        t_request_start = time.perf_counter()
+
+        # Stage 1: prompt build
+        t0 = time.perf_counter()
+        prompt = self.prompt_builder.build_prefix(speaker, text)
+        perf["prompt_build_ms"] = (time.perf_counter() - t0) * 1000
+
+        # Prompt token count (for TPS normalization)
+        try:
+            perf["llm_prompt_token_total"] = len(
+                self.model.tokenizer.encode(prompt, add_special_tokens=False)
+            )
+        except Exception:
+            perf["llm_prompt_token_total"] = 0
+
+        # Stage 2: sampling params
+        t0 = time.perf_counter()
+        sampling_params = SamplingParams(
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            max_tokens=max_tokens,
+            repetition_penalty=repetition_penalty,
+            stop=TRAINING_STOP_TOKEN_IDS,
+            skip_special_tokens=False,
+            seed=seed,
+        )
+        perf["sampling_params_ms"] = (time.perf_counter() - t0) * 1000
+
+        request_id = f"req-{uuid.uuid4().hex[:12]}"
+        results_generator = self.model.engine.generate(
+            prompt=prompt,
+            sampling_params=sampling_params,
+            request_id=request_id,
+        )
+
+        final_output = None
+        final_request_metrics: Dict[str, float] = {}
+
+        llm_batch_wall_ms: List[float] = []
+        llm_batch_gpu_ms: List[float] = []
+        llm_decode_wall_ms: List[float] = []
+        llm_decode_gpu_ms: List[float] = []
+        tokens_per_batch: List[float] = []
+
+        prev_generated_count = 0
+        prev_gpu_cumulative_s: Optional[float] = None
+
+        t_gen_start = time.perf_counter()
+        t_last_yield = t_gen_start
+        async for request_output in results_generator:
+            now = time.perf_counter()
+            batch_wall = (now - t_last_yield) * 1000
+            t_last_yield = now
+
+            outputs = request_output.outputs or []
+            if not outputs:
+                final_output = request_output
+                continue
+
+            output = outputs[0]
+            generated_count = len(output.token_ids)
+            new_tokens = max(0, generated_count - prev_generated_count)
+            prev_generated_count = generated_count
+
+            raw_req_metrics = self._extract_request_metrics(getattr(request_output, "metrics", None))
+            if raw_req_metrics:
+                final_request_metrics = raw_req_metrics
+
+            if new_tokens <= 0:
+                final_output = request_output
+                continue
+
+            perf["llm_batch_count"] += 1
+            tokens_per_batch.append(float(new_tokens))
+            llm_batch_wall_ms.append(float(batch_wall))
+            if perf["llm_batch_count"] > 1:
+                llm_decode_wall_ms.append(float(batch_wall))
+
+            cumulative_gpu_s: Optional[float] = None
+            if raw_req_metrics:
+                if "model_execute_time" in raw_req_metrics:
+                    cumulative_gpu_s = raw_req_metrics["model_execute_time"]
+                elif "model_forward_time" in raw_req_metrics:
+                    cumulative_gpu_s = raw_req_metrics["model_forward_time"]
+
+            if cumulative_gpu_s is not None:
+                if prev_gpu_cumulative_s is None:
+                    gpu_delta_ms = cumulative_gpu_s * 1000
+                elif cumulative_gpu_s >= prev_gpu_cumulative_s:
+                    gpu_delta_ms = (cumulative_gpu_s - prev_gpu_cumulative_s) * 1000
+                else:
+                    # Some versions report non-cumulative per-step values.
+                    gpu_delta_ms = cumulative_gpu_s * 1000
+                prev_gpu_cumulative_s = cumulative_gpu_s
+
+                if gpu_delta_ms >= 0:
+                    llm_batch_gpu_ms.append(float(gpu_delta_ms))
+                    if perf["llm_batch_count"] > 1:
+                        llm_decode_gpu_ms.append(float(gpu_delta_ms))
+
+            final_output = request_output
+
+        perf["llm_generation_wall_ms"] = (time.perf_counter() - t_gen_start) * 1000
+
+        if final_output is None:
+            perf["pipeline_total_ms"] = (time.perf_counter() - t_request_start) * 1000
+            return None, perf
+
+        generated_text = final_output.outputs[0].text
+        generated_ids = final_output.outputs[0].token_ids
+        perf["llm_token_total"] = len(generated_ids)
+
+        # Stage 3: parse BiCodec tokens from generated text
+        t0 = time.perf_counter()
+        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)
+        perf["llm_parse_ms"] = (time.perf_counter() - t0) * 1000
+        perf["semantic_token_total"] = len(semantic_ids)
+        perf["global_token_total"] = len(global_ids)
+
+        # Aggregate LLM batch stats
+        self._apply_stats(perf, "llm_batch_wall_ms", llm_batch_wall_ms)
+        self._apply_stats(perf, "llm_batch_gpu_ms", llm_batch_gpu_ms)
+        self._apply_stats(perf, "llm_decode_wall_ms", llm_decode_wall_ms)
+        self._apply_stats(perf, "llm_decode_gpu_ms", llm_decode_gpu_ms)
+        self._apply_stats(perf, "tokens_per_batch", tokens_per_batch)
+
+        if llm_batch_wall_ms:
+            perf["llm_prefill_wall_ms"] = float(llm_batch_wall_ms[0])
+            perf["llm_prefill_gpu_ms"] = float(llm_batch_gpu_ms[0]) if llm_batch_gpu_ms else 0.0
+        else:
+            perf["llm_prefill_wall_ms"] = float(perf["llm_generation_wall_ms"])
+            perf["llm_prefill_gpu_ms"] = 0.0
+
+        if not llm_batch_gpu_ms and final_request_metrics.get("model_execute_time") is not None:
+            total_gpu_ms = float(final_request_metrics["model_execute_time"]) * 1000
+            perf["llm_batch_gpu_ms_total"] = total_gpu_ms
+            if perf["llm_batch_count"] > 1:
+                perf["llm_decode_gpu_ms_total"] = total_gpu_ms
+
+        if perf["llm_token_total"] > 0:
+            perf["llm_time_per_token_ms"] = float(perf["llm_generation_wall_ms"]) / float(perf["llm_token_total"])
+        if perf["llm_batch_count"] > 0:
+            perf["llm_time_per_batch_wall_ms"] = float(perf.get("llm_batch_wall_ms_total", 0.0)) / float(perf["llm_batch_count"])
+            if perf.get("llm_batch_gpu_ms_total", 0.0) > 0:
+                perf["llm_time_per_batch_gpu_ms"] = float(perf["llm_batch_gpu_ms_total"]) / float(perf["llm_batch_count"])
+
+        # Pull useful request-level vLLM metrics (seconds -> ms)
+        sec_fields = {
+            "time_in_queue": "llm_time_in_queue_ms",
+            "scheduler_time": "llm_scheduler_ms",
+            "model_forward_time": "llm_model_forward_ms",
+            "model_execute_time": "llm_model_execute_ms",
+        }
+        for src, dst in sec_fields.items():
+            if src in final_request_metrics:
+                perf[dst] = float(final_request_metrics[src]) * 1000
+
+        arrival = final_request_metrics.get("arrival_time")
+        first_token = final_request_metrics.get("first_token_time")
+        finished = final_request_metrics.get("finished_time")
+        if arrival is not None and first_token is not None and first_token >= arrival:
+            perf["llm_first_token_ms"] = (first_token - arrival) * 1000
+        if arrival is not None and finished is not None and finished >= arrival:
+            perf["llm_request_lifecycle_ms"] = (finished - arrival) * 1000
+
+        if not semantic_ids or not global_ids:
+            perf["pipeline_total_ms"] = (time.perf_counter() - t_request_start) * 1000
+            return None, perf
+
+        # Stage 4: validate BiCodec tokens
+        t0 = time.perf_counter()
+        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
+            perf["token_validation_ms"] = (time.perf_counter() - t0) * 1000
+            perf["pipeline_total_ms"] = (time.perf_counter() - t_request_start) * 1000
+            return None, perf
+        perf["token_validation_ms"] = (time.perf_counter() - t0) * 1000
+
+        # Stage 5: BiCodec decode (wall + GPU/CPU split)
+        decode_t0 = time.perf_counter()
+        decode_gpu_ms = 0.0
+        perf["llm_decode_calls"] = 1
+
+        use_cuda_timing = (
+            torch.cuda.is_available()
+            and hasattr(self.bicodec_decoder, "device")
+            and getattr(self.bicodec_decoder.device, "type", "") == "cuda"
+        )
+        audio_bytes: Optional[bytes]
+        if use_cuda_timing:
+            try:
+                device = self.bicodec_decoder.device
+                start_event = torch.cuda.Event(enable_timing=True)
+                end_event = torch.cuda.Event(enable_timing=True)
+                torch.cuda.synchronize(device=device)
+                start_event.record()
+                audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
+                end_event.record()
+                torch.cuda.synchronize(device=device)
+                decode_gpu_ms = float(start_event.elapsed_time(end_event))
+            except Exception:
+                audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
+                decode_gpu_ms = 0.0
+        else:
+            audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
+
+        decode_wall_ms = (time.perf_counter() - decode_t0) * 1000
+        perf["bicodec_decode_wall_ms"] = float(decode_wall_ms)
+        perf["bicodec_decode_gpu_ms"] = float(max(0.0, decode_gpu_ms))
+        perf["bicodec_decode_cpu_ms"] = float(max(0.0, decode_wall_ms - decode_gpu_ms))
+
+        if audio_bytes is None:
+            perf["pipeline_total_ms"] = (time.perf_counter() - t_request_start) * 1000
+            return None, perf
+
+        # Stage 6: WAV header
+        t0 = time.perf_counter()
+        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)
+        perf["wav_pack_ms"] = (time.perf_counter() - t0) * 1000
+
+        perf["pipeline_total_ms"] = (time.perf_counter() - t_request_start) * 1000
+        return wav_bytes, perf

     async def generate_speech(
         self,
         speaker: str,
         text: str,
@@
         Returns:
             Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed
         """
-        # Build prompt using Spark TTS format
-        prompt = self.prompt_builder.build_prefix(speaker, text)
-
-        # Configure sampling for Spark TTS
-        sampling_params = SamplingParams(
-            temperature=temperature,
-            top_k=top_k,
-            top_p=top_p,
-            max_tokens=max_tokens,
-            repetition_penalty=repetition_penalty,  # Prevent token repetition (e.g., token 484 loop)
-            stop=TRAINING_STOP_TOKEN_IDS,  # "<|im_end|>"
-            skip_special_tokens=False,  # Keep special tokens for BiCodec extraction
-            seed=seed,
-        )
-
-        # Generate tokens
-        import uuid
-        request_id = f"req-{uuid.uuid4().hex[:12]}"
-
-        # Use vLLM engine to generate
-        results_generator = self.model.engine.generate(
-            prompt=prompt,
-            sampling_params=sampling_params,
-            request_id=request_id,
-        )
-
-        # Collect all outputs
-        final_output = None
-        async for request_output in results_generator:
-            final_output = request_output
-
-        if final_output is None:
-            print(f"❌ Generation failed: no output")
-            return None
-
-        # Extract generated text (contains BiCodec tokens)
-        generated_text = final_output.outputs[0].text
-        generated_ids = final_output.outputs[0].token_ids
-
-        # Extract BiCodec tokens using regex
-        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)
-
-        if not semantic_ids or not global_ids:
-            logger.error(
-                "❌ No BiCodec tokens found in output",
-                extra={
-                    "speaker": speaker,
-                    "text_length": len(text),
-                    "text_preview": text[:100],
-                    "generated_tokens": len(generated_ids),
-                    "semantic_tokens": len(semantic_ids),
-                    "global_tokens": len(global_ids),
-                    "generated_text_preview": generated_text[:500],
-                }
-            )
-            return None
-
-        # Validate tokens
-        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
-            logger.error("❌ BiCodec token validation failed", extra={
-                "speaker": speaker,
-                "text_length": len(text),
-                "semantic_tokens": len(semantic_ids),
-                "global_tokens": len(global_ids),
-            })
-            return None
-
-        # Decode to audio
-        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
-
-        if audio_bytes is None:
-            logger.error("❌ BiCodec decode failed", extra={
-                "speaker": speaker,
-                "text_length": len(text),
-            })
-            return None
-
-        # Add WAV header
-        from veena3modal.audio.utils import add_wav_header
-        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)
-
-        return wav_bytes
+        wav_bytes, _ = await self.generate_speech_profiled(
+            speaker=speaker,
+            text=text,
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            max_tokens=max_tokens,
+            repetition_penalty=repetition_penalty,
+            seed=seed,
+        )
+        return wav_bytes
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Designing speech generation metrics integration**


I’ve finished the pipeline instrumentation and now I’m wiring it into `tts_runtime` so each non-streaming request carries the detailed timing payload all the way back to the API layer (including the non-chunked branch inside chunking mode).


Input:
*** Begin Patch
*** Update File: veena3modal/services/tts_runtime.py
@@
         # Generate audio using pipeline (always at 16kHz)
-        audio_bytes = await runtime.pipeline.generate_speech_indic(
+        audio_bytes, perf = await runtime.pipeline.generate_speech_profiled(
             speaker=speaker,
             text=text,
             temperature=temperature,
             top_k=top_k,
             top_p=top_p,
             max_tokens=max_tokens,
             repetition_penalty=repetition_penalty,
             seed=seed,
         )
+        if perf:
+            metrics.update(perf)

         generation_time = time.time() - start_time
         metrics["generation_ms"] = int(generation_time * 1000)
         metrics["ttfb_ms"] = metrics["generation_ms"]  # Non-streaming: TTFB = total time
@@
         # Check if chunking is needed
         if long_processor.should_chunk(text):
             metrics["text_chunked"] = True
             audio_bytes = await long_processor.generate_with_chunking(
                 text=text,
@@
         else:
             # Short text: direct generation
-            audio_bytes = await runtime.pipeline.generate_speech_indic(
+            audio_bytes, perf = await runtime.pipeline.generate_speech_profiled(
                 speaker=speaker,
                 text=text,
                 temperature=temperature,
                 top_k=top_k,
                 top_p=top_p,
                 max_tokens=max_tokens,
                 repetition_penalty=repetition_penalty,
                 seed=seed,
             )
+            if perf:
+                metrics.update(perf)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/services/tts_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Adding inflight tracking and perf headers**


Input:
*** Begin Patch
*** Update File: veena3modal/api/fastapi_app.py
@@
 import time
 import uuid
-from typing import Optional
+import json
+import threading
+from typing import Optional, Any, Dict
@@
 _MODEL_VERSION: Optional[str] = None
 _startup_time: Optional[float] = None
+
+# Request-level in-flight tracking (single-process scope)
+_inflight_lock = threading.Lock()
+_inflight_requests = 0
+
+
+def _inflight_enter() -> int:
+    """Increment in-flight counter and return current value."""
+    global _inflight_requests
+    with _inflight_lock:
+        _inflight_requests += 1
+        return _inflight_requests
+
+
+def _inflight_exit() -> int:
+    """Decrement in-flight counter and return current value."""
+    global _inflight_requests
+    with _inflight_lock:
+        _inflight_requests = max(0, _inflight_requests - 1)
+        return _inflight_requests
+
+
+def _serialize_header_value(value: Any) -> Optional[str]:
+    """Convert metric values to compact header-safe strings."""
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        if value.is_integer():
+            return str(int(value))
+        return f"{value:.3f}"
+    return str(value)
+
+
+def _build_perf_headers(metrics: Dict[str, Any], request_inflight: int) -> Dict[str, str]:
+    """
+    Build detailed timing headers consumed by local benchmark scripts.
+
+    Includes a compact JSON payload (`X-Perf-Details`) and selected explicit headers
+    for easy parsing in shell scripts/tools.
+    """
+    perf_headers: Dict[str, str] = {}
+    if not isinstance(metrics, dict):
+        metrics = {}
+
+    detail_payload: Dict[str, Any] = {}
+    if request_inflight > 0:
+        detail_payload["request_inflight"] = request_inflight
+
+    # Expose selected metrics as explicit headers.
+    header_map = {
+        "x-generation-ms": "generation_ms",
+        "x-request-inflight": "request_inflight",
+        "x-llm-token-total": "llm_token_total",
+        "x-llm-batch-count": "llm_batch_count",
+        "x-llm-batch-wall-ms": "llm_batch_wall_ms_total",
+        "x-llm-batch-gpu-ms": "llm_batch_gpu_ms_total",
+        "x-llm-batch-wall-ms-min": "llm_batch_wall_ms_min",
+        "x-llm-batch-wall-ms-max": "llm_batch_wall_ms_max",
+        "x-llm-batch-wall-ms-p50": "llm_batch_wall_ms_p50",
+        "x-llm-batch-gpu-ms-min": "llm_batch_gpu_ms_min",
+        "x-llm-batch-gpu-ms-max": "llm_batch_gpu_ms_max",
+        "x-llm-batch-gpu-ms-p50": "llm_batch_gpu_ms_p50",
+        "x-llm-decode-wall-ms": "llm_decode_wall_ms_total",
+        "x-llm-decode-gpu-ms": "llm_decode_gpu_ms_total",
+        "x-llm-decode-wall-ms-min": "llm_decode_wall_ms_min",
+        "x-llm-decode-wall-ms-max": "llm_decode_wall_ms_max",
+        "x-llm-decode-wall-ms-p50": "llm_decode_wall_ms_p50",
+        "x-llm-decode-gpu-ms-min": "llm_decode_gpu_ms_min",
+        "x-llm-decode-gpu-ms-max": "llm_decode_gpu_ms_max",
+        "x-llm-decode-gpu-ms-p50": "llm_decode_gpu_ms_p50",
+        "x-llm-decode-calls": "llm_decode_calls",
+        "x-llm-decode-cpu-ms": "bicodec_decode_cpu_ms",
+        "x-llm-parse-ms": "llm_parse_ms",
+        "x-llm-parse-avg-ms": "llm_parse_ms",
+        "x-llm-tokens-per-batch-min": "tokens_per_batch_min",
+        "x-llm-tokens-per-batch-max": "tokens_per_batch_max",
+        "x-llm-tokens-per-batch-p50": "tokens_per_batch_p50",
+        "x-llm-time-per-token-ms": "llm_time_per_token_ms",
+        "x-llm-time-per-batch-wall-ms": "llm_time_per_batch_wall_ms",
+        "x-llm-time-per-batch-gpu-ms": "llm_time_per_batch_gpu_ms",
+        "x-llm-text-chunked": "text_chunked",
+        "x-llm-chunks-processed": "chunks_processed",
+        "x-bicodec-decode-wall-ms": "bicodec_decode_wall_ms",
+        "x-bicodec-decode-gpu-ms": "bicodec_decode_gpu_ms",
+        "x-bicodec-decode-cpu-ms": "bicodec_decode_cpu_ms",
+    }
+
+    if request_inflight > 0:
+        metrics = dict(metrics)
+        metrics["request_inflight"] = request_inflight
+
+    for header_key, metric_key in header_map.items():
+        if metric_key not in metrics:
+            continue
+        value = metrics.get(metric_key)
+        serialized = _serialize_header_value(value)
+        if serialized is None:
+            continue
+        perf_headers[header_key] = serialized
+        detail_payload[metric_key] = value
+
+    # Include additional timing fields (bounded set) in JSON payload only.
+    extra_detail_keys = (
+        "prompt_build_ms",
+        "sampling_params_ms",
+        "llm_prefill_wall_ms",
+        "llm_prefill_gpu_ms",
+        "llm_generation_wall_ms",
+        "llm_time_in_queue_ms",
+        "llm_scheduler_ms",
+        "llm_model_forward_ms",
+        "llm_model_execute_ms",
+        "llm_first_token_ms",
+        "llm_request_lifecycle_ms",
+        "llm_prompt_token_total",
+        "semantic_token_total",
+        "global_token_total",
+        "wav_pack_ms",
+        "pipeline_total_ms",
+        "token_validation_ms",
+    )
+    for key in extra_detail_keys:
+        if key in metrics:
+            detail_payload[key] = metrics[key]
+
+    perf_headers["x-perf-details"] = json.dumps(
+        detail_payload,
+        separators=(",", ":"),
+        ensure_ascii=True,
+    )
+    return perf_headers
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/api/fastapi_app.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Planning precise patch location**


nl -ba veena3modal/api/fastapi_app.py | sed -n '220,520p'


Output:
Chunk ID: 52693a
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 3427
Output:
   220	def set_model_version(version: str) -> None:
   221	    """Set the model version after loading."""
   222	    global _MODEL_VERSION
   223	    _MODEL_VERSION = version
   224
   225
   226	def create_app():
   227	    """
   228	    Factory function to create FastAPI app.
   229
   230	    Import inside factory so local tooling can import this module without FastAPI installed.
   231	    """
   232	    from fastapi import FastAPI, HTTPException, Request
   233	    from fastapi.responses import JSONResponse, Response
   234
   235	    global _startup_time
   236	    _startup_time = time.time()
   237
   238	    app = FastAPI(
   239	        title="Veena3 TTS (Modal)",
   240	        version=_APP_VERSION,
   241	        description="High-quality multilingual Text-to-Speech API with true streaming support.",
   242	    )
   243
   244	    @app.get("/v1/tts/health")
   245	    async def tts_health():
   246	        """
   247	        Health check endpoint.
   248
   249	        Returns:
   250	            - status: "healthy" | "degraded" | "unhealthy"
   251	            - model_loaded: whether TTS model is initialized
   252	            - model_version: version string of loaded model
   253	            - uptime_seconds: time since app startup
   254	            - gpu_available: whether GPU is detected (best-effort check)
   255	        """
   256	        # Check model status via runtime
   257	        from veena3modal.services import tts_runtime
   258	        model_loaded = tts_runtime.is_initialized()
   259
   260	        # Use runtime's model version if available
   261	        runtime = tts_runtime.get_runtime()
   262	        model_version = runtime.model_version if runtime else get_model_version()
   263
   264	        # Check GPU availability (best-effort, non-blocking)
   265	        gpu_available = False
   266	        try:
   267	            import torch
   268	            gpu_available = torch.cuda.is_available()
   269	        except ImportError:
   270	            pass  # torch not installed, skip GPU check
   271
   272	        # Compute uptime
   273	        uptime = time.time() - _startup_time if _startup_time else 0
   274
   275	        # Determine overall status
   276	        if model_loaded and gpu_available:
   277	            status = "healthy"
   278	        elif model_loaded or gpu_available:
   279	            status = "degraded"
   280	        else:
   281	            status = "degraded"
   282
   283	        return JSONResponse(
   284	            content={
   285	                "status": status,
   286	                "model_loaded": model_loaded,
   287	                "model_version": model_version,
   288	                "uptime_seconds": round(uptime, 2),
   289	                "gpu_available": gpu_available,
   290	                "app_version": _APP_VERSION,
   291	            },
   292	            headers={
   293	                "X-Model-Version": model_version,
   294	                "X-App-Version": _APP_VERSION,
   295	            },
   296	        )
   297
   298	    @app.get("/v1/tts/metrics")
   299	    async def tts_metrics():
   300	        """
   301	        Prometheus metrics endpoint.
   302
   303	        Returns metrics in Prometheus text format for scraping.
   304	        """
   305	        return Response(
   306	            content=get_metrics_text(),
   307	            media_type=get_metrics_content_type(),
   308	        )
   309
   310	    @app.post("/v1/tts/generate")
   311	    async def tts_generate(request: Request):
   312	        """
   313	        Generate speech from text.
   314
   315	        Supports streaming (M4) and non-streaming (M3) modes.
   316	        Currently implements non-streaming WAV generation only.
   317	        """
   318	        from veena3modal.api.schemas import TTSGenerateRequest, AudioFormat
   319	        from veena3modal.services import tts_runtime
   320
   321	        request_id = str(uuid.uuid4())
   322	        start_time = time.time()
   323
   324	        # === AUTH CHECK ===
   325	        if FeatureFlags.is_auth_enabled():
   326	            # Extract API key from headers
   327	            headers_dict = dict(request.headers)
   328	            api_key = extract_api_key(headers_dict)
   329
   330	            if not api_key:
   331	                return JSONResponse(
   332	                    status_code=401,
   333	                    content=create_error_response(
   334	                        code=ErrorCode.INVALID_API_KEY,
   335	                        message="API key required. Use 'Authorization: Bearer <key>' or 'X-API-Key: <key>' header.",
   336	                        request_id=request_id,
   337	                    ),
   338	                    headers={"X-Request-ID": request_id},
   339	                )
   340
   341	            # Validate API key
   342	            validator = get_api_validator()
   343	            key_hash = hash_api_key(api_key)
   344	            auth_result = validator.validate(key_hash)
   345
   346	            if not auth_result.is_valid:
   347	                status_code = 401 if auth_result.error_code == "INVALID_API_KEY" else 403
   348	                return JSONResponse(
   349	                    status_code=status_code,
   350	                    content=create_error_response(
   351	                        code=ErrorCode(auth_result.error_code),
   352	                        message=auth_result.error_message or "Authentication failed",
   353	                        request_id=request_id,
   354	                    ),
   355	                    headers={"X-Request-ID": request_id},
   356	                )
   357
   358	            # === RATE LIMIT CHECK ===
   359	            if FeatureFlags.is_rate_limiting_enabled():
   360	                rate_limiter = get_rate_limiter()
   361	                allowed, remaining, reset_after = rate_limiter.check(key_hash)
   362
   363	                if not allowed:
   364	                    headers = rate_limiter.get_headers(allowed, remaining, reset_after)
   365	                    headers["X-Request-ID"] = request_id
   366	                    return JSONResponse(
   367	                        status_code=429,
   368	                        content=create_error_response(
   369	                            code=ErrorCode.RATE_LIMIT_EXCEEDED,
   370	                            message=f"Rate limit exceeded. Retry after {int(reset_after) + 1} seconds.",
   371	                            request_id=request_id,
   372	                        ),
   373	                        headers=headers,
   374	                    )
   375
   376	        # Parse and validate request body
   377	        try:
   378	            body = await request.json()
   379	        except Exception as e:
   380	            raise HTTPException(status_code=400, detail=f"Invalid JSON: {e}")
   381
   382	        try:
   383	            req = TTSGenerateRequest(**body)
   384	        except Exception as e:
   385	            error_msg = str(e)
   386	            # Extract validation error details
   387	            return JSONResponse(
   388	                status_code=400,
   389	                content={
   390	                    "error": {
   391	                        "code": "VALIDATION_ERROR",
   392	                        "message": error_msg,
   393	                        "request_id": request_id,
   394	                    }
   395	                },
   396	                headers={"X-Request-ID": request_id},
   397	            )
   398
   399	        # Check if runtime is initialized
   400	        if not tts_runtime.is_initialized():
   401	            return JSONResponse(
   402	                status_code=503,
   403	                content={
   404	                    "error": {
   405	                        "code": "MODEL_NOT_LOADED",
   406	                        "message": "TTS model not initialized. Please wait for warmup.",
   407	                        "request_id": request_id,
   408	                    }
   409	                },
   410	                headers={"X-Request-ID": request_id},
   411	            )
   412
   413	        # Get resolved speaker
   414	        speaker = req.get_resolved_speaker()
   415
   416	        # Get normalized text (apply text normalization if enabled)
   417	        normalizer_func = None
   418	        if req.normalize:
   419	            try:
   420	                from veena3modal.processing.text_normalizer import normalize_text
   421	                normalizer_func = lambda t: normalize_text(t, verbose=req.normalize_verbose)
   422	            except ImportError:
   423	                logger.warning("TextNormalizer not available, skipping text normalization")
   424
   425	        text = req.get_normalized_text(normalizer_func=normalizer_func)
   426
   427	        # Set request context for structured logging
   428	        set_request_context(request_id=request_id)
   429
   430	        # Log request received (lifecycle event + metrics)
   431	        log_request_received(
   432	            request_id=request_id,
   433	            text_length=len(text),
   434	            speaker=speaker,
   435	            stream=req.stream,
   436	            format=req.format,
   437	        )
   438	        metrics_request_received(speaker=speaker, stream=req.stream, format=req.format)
   439
   440	        # Streaming mode (M4)
   441	        if req.stream:
   442	            return await _handle_streaming_request(req, request_id, text, speaker, start_time)
   443
   444	        # Format encoding will be applied after generation (non-streaming only)
   445	        # For streaming, only WAV is supported
   446	        target_format = req.format
   447
   448	        try:
   449	            # Get output sample rate - use_enum_values=True means req.output is already a string
   450	            output_sr = req.output if hasattr(req, 'output') else "16khz"
   451
   452	            # Generate speech (non-streaming)
   453	            if req.chunking:
   454	                audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
   455	                    text=text,
   456	                    speaker=speaker,
   457	                    temperature=req.temperature,
   458	                    top_k=req.top_k,
   459	                    top_p=req.top_p,
   460	                    max_tokens=req.max_tokens,
   461	                    repetition_penalty=req.repetition_penalty,
   462	                    seed=req.seed,
   463	                    sample_rate=req.sample_rate,
   464	                    output_sample_rate=output_sr,
   465	                )
   466	            else:
   467	                audio_bytes, metrics = await tts_runtime.generate_speech(
   468	                    text=text,
   469	                    speaker=speaker,
   470	                    temperature=req.temperature,
   471	                    top_k=req.top_k,
   472	                    top_p=req.top_p,
   473	                    max_tokens=req.max_tokens,
   474	                    repetition_penalty=req.repetition_penalty,
   475	                    seed=req.seed,
   476	                    output_sample_rate=output_sr,
   477	                )
   478
   479	            if audio_bytes is None:
   480	                return JSONResponse(
   481	                    status_code=500,
   482	                    content={
   483	                        "error": {
   484	                            "code": "GENERATION_FAILED",
   485	                            "message": "Audio generation failed. Check logs for details.",
   486	                            "request_id": request_id,
   487	                        }
   488	                    },
   489	                    headers={"X-Request-ID": request_id},
   490	                )
   491
   492	            # Calculate RTF
   493	            total_time = time.time() - start_time
   494	            audio_duration = metrics.get("audio_duration_seconds", 0)
   495	            rtf = total_time / audio_duration if audio_duration > 0 else 0
   496
   497	            # Build response headers
   498	            runtime = tts_runtime.get_runtime()
   499	            model_version = runtime.model_version if runtime else get_model_version()
   500
   501	            # Use actual output sample rate from metrics
   502	            output_sample_rate = metrics.get("output_sample_rate", req.sample_rate)
   503
   504	            headers = {
   505	                "X-Request-ID": request_id,
   506	                "X-Model-Version": model_version,
   507	                "X-Format": "wav",
   508	                "X-Sample-Rate": str(output_sample_rate),
   509	                "X-Stream": "false",
   510	                "X-Audio-Bytes": str(len(audio_bytes)),
   511	                "X-Audio-Seconds": f"{audio_duration:.2f}",
   512	                "X-TTFB-ms": str(metrics.get("ttfb_ms", 0)),
   513	                "X-RTF": f"{rtf:.3f}",
   514	                "X-Text-Chunked": str(metrics.get("text_chunked", False)).lower(),
   515	                "X-SR-Applied": str(metrics.get("sr_applied", False)).lower(),
   516	            }
   517
   518	            if req.seed is not None:
   519	                headers["X-Seed"] = str(req.seed)
   520


nl -ba veena3modal/api/fastapi_app.py | sed -n '520,760p'


Output:
Chunk ID: fb9f72
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 3055
Output:
   520
   521	            # Fire-and-forget sentence storage (non-blocking, after generation)
   522	            sentence_store = _get_sentence_store()
   523	            sentence_store.store_fire_and_forget(
   524	                request_id=request_id,
   525	                text=text,
   526	                speaker=speaker,
   527	                stream=False,
   528	                format=req.format,
   529	                temperature=req.temperature,
   530	                top_k=req.top_k,
   531	                top_p=req.top_p,
   532	                max_tokens=req.max_tokens,
   533	                repetition_penalty=req.repetition_penalty,
   534	                seed=req.seed,
   535	                text_chunked=metrics.get("text_chunked", False),
   536	                ttfb_ms=metrics.get("ttfb_ms"),
   537	                audio_duration_seconds=audio_duration,
   538	            )
   539
   540	            # Log successful completion (lifecycle event + metrics)
   541	            total_duration_ms = int(total_time * 1000)
   542	            log_request_completed(
   543	                request_id=request_id,
   544	                status_code=200,
   545	                total_duration_ms=total_duration_ms,
   546	                audio_duration_seconds=audio_duration,
   547	                rtf=rtf,
   548	                chunks_sent=1,
   549	            )
   550	            metrics_request_completed(
   551	                status_code=200,
   552	                duration_seconds=total_time,
   553	                speaker=speaker,
   554	                stream=False,
   555	            )
   556	            record_ttfb(ttfb_seconds=metrics.get("ttfb_ms", 0) / 1000, speaker=speaker, stream=False)
   557	            record_rtf(rtf=rtf, speaker=speaker)
   558	            record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
   559
   560	            clear_request_context()
   561
   562	            # Apply format encoding if not WAV
   563	            media_type = "audio/wav"
   564	            if target_format != AudioFormat.WAV.value:
   565	                try:
   566	                    from veena3modal.audio.encoder import (
   567	                        OpusEncoder, MP3Encoder, MuLawEncoder, FLACEncoder
   568	                    )
   569
   570	                    # Get sample rate from output (after potential SR)
   571	                    sample_rate = metrics.get("output_sample_rate", 16000)
   572
   573	                    # Strip WAV header to get PCM data
   574	                    pcm_data = audio_bytes[44:] if len(audio_bytes) > 44 else audio_bytes
   575
   576	                    if target_format == AudioFormat.OPUS.value:
   577	                        encoder = OpusEncoder(sample_rate=sample_rate)
   578	                        audio_bytes = encoder.encode(pcm_data, sample_rate)
   579	                        media_type = "audio/opus"
   580	                    elif target_format == AudioFormat.MP3.value:
   581	                        encoder = MP3Encoder(sample_rate=sample_rate)
   582	                        audio_bytes = encoder.encode(pcm_data, sample_rate)
   583	                        media_type = "audio/mpeg"
   584	                    elif target_format == AudioFormat.MULAW.value:
   585	                        encoder = MuLawEncoder()
   586	                        audio_bytes = encoder.encode(pcm_data, sample_rate)
   587	                        media_type = "audio/x-wav"
   588	                    elif target_format == AudioFormat.FLAC.value:
   589	                        encoder = FLACEncoder(sample_rate=sample_rate)
   590	                        audio_bytes = encoder.encode(pcm_data, sample_rate)
   591	                        media_type = "audio/flac"
   592
   593	                    headers["X-Audio-Bytes"] = str(len(audio_bytes))
   594	                    logger.info(f"Encoded audio to {target_format}: {len(audio_bytes)} bytes")
   595	                except ImportError as e:
   596	                    logger.warning(f"Audio encoder not available: {e}")
   597	                    # Fall back to WAV
   598	                except Exception as e:
   599	                    logger.error(f"Format encoding failed: {e}")
   600	                    # Fall back to WAV
   601
   602	            return Response(
   603	                content=audio_bytes,
   604	                media_type=media_type,
   605	                headers=headers,
   606	            )
   607
   608	        except Exception as e:
   609	            logger.exception(f"TTS generation error: {e}")
   610
   611	            # Log failure (lifecycle event + metrics)
   612	            log_request_failed(
   613	                request_id=request_id,
   614	                status_code=500,
   615	                error_code="INTERNAL_ERROR",
   616	                error_message=str(e),
   617	            )
   618	            metrics_request_failed(
   619	                status_code=500,
   620	                error_code="INTERNAL_ERROR",
   621	                speaker=speaker,
   622	            )
   623	            clear_request_context()
   624
   625	            return JSONResponse(
   626	                status_code=500,
   627	                content={
   628	                    "error": {
   629	                        "code": "INTERNAL_ERROR",
   630	                        "message": str(e),
   631	                        "request_id": request_id,
   632	                    }
   633	                },
   634	                headers={"X-Request-ID": request_id},
   635	            )
   636
   637	    async def _handle_streaming_request(req, request_id: str, text: str, speaker: str, start_time: float):
   638	        """
   639	        Handle streaming TTS request.
   640
   641	        Returns a StreamingResponse that yields WAV header + PCM chunks.
   642	        True streaming: first bytes sent ASAP before full audio is generated.
   643	        """
   644	        from fastapi.responses import StreamingResponse
   645	        from veena3modal.api.schemas import AudioFormat
   646	        from veena3modal.services import tts_runtime
   647
   648	        # Check if streaming pipeline is available
   649	        runtime = tts_runtime.get_runtime()
   650	        if runtime is None or runtime.streaming_pipeline is None:
   651	            return JSONResponse(
   652	                status_code=503,
   653	                content={
   654	                    "error": {
   655	                        "code": "STREAMING_UNAVAILABLE",
   656	                        "message": "Streaming pipeline not initialized.",
   657	                        "request_id": request_id,
   658	                    }
   659	                },
   660	                headers={"X-Request-ID": request_id},
   661	            )
   662
   663	        # Non-WAV formats not implemented yet (M5)
   664	        if req.format != AudioFormat.WAV.value:
   665	            return JSONResponse(
   666	                status_code=501,
   667	                content={
   668	                    "error": {
   669	                        "code": "FORMAT_NOT_IMPLEMENTED",
   670	                        "message": f"Streaming format '{req.format}' not yet implemented. Use format=wav.",
   671	                        "request_id": request_id,
   672	                    }
   673	                },
   674	                headers={"X-Request-ID": request_id},
   675	            )
   676
   677	        # Set request context for structured logging
   678	        set_request_context(request_id=request_id)
   679
   680	        # Create async generator for streaming response
   681	        async def audio_stream_generator():
   682	            """Yields audio chunks as they're generated."""
   683	            final_metrics = {}
   684	            first_chunk_sent = False
   685	            chunks_count = 0
   686	            try:
   687	                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
   688	                    text=text,
   689	                    speaker=speaker,
   690	                    temperature=req.temperature,
   691	                    top_k=req.top_k,
   692	                    top_p=req.top_p,
   693	                    max_tokens=req.max_tokens,
   694	                    repetition_penalty=req.repetition_penalty,
   695	                    seed=req.seed,
   696	                    enable_chunking=req.chunking,
   697	                ):
   698	                    final_metrics = metrics
   699	                    chunks_count += 1
   700
   701	                    # Fire-and-forget sentence storage after first chunk (don't block TTFB)
   702	                    if not first_chunk_sent:
   703	                        first_chunk_sent = True
   704	                        ttfb_ms = metrics.get("ttfb_ms", 0)
   705
   706	                        # Log first audio emitted (TTFB marker)
   707	                        log_first_audio_emitted(
   708	                            request_id=request_id,
   709	                            ttfb_ms=ttfb_ms,
   710	                            chunk_size_bytes=len(audio_chunk),
   711	                        )
   712	                        record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)
   713
   714	                        sentence_store = _get_sentence_store()
   715	                        sentence_store.store_fire_and_forget(
   716	                            request_id=request_id,
   717	                            text=text,
   718	                            speaker=speaker,
   719	                            stream=True,
   720	                            format=req.format,
   721	                            temperature=req.temperature,
   722	                            top_k=req.top_k,
   723	                            top_p=req.top_p,
   724	                            max_tokens=req.max_tokens,
   725	                            repetition_penalty=req.repetition_penalty,
   726	                            seed=req.seed,
   727	                            text_chunked=metrics.get("text_chunked", False),
   728	                            ttfb_ms=ttfb_ms,
   729	                            # audio_duration_seconds not known yet for streaming
   730	                        )
   731
   732	                    yield audio_chunk
   733
   734	                # Stream completed - log metrics
   735	                total_time = time.time() - start_time
   736	                audio_duration = final_metrics.get("audio_duration_seconds", 0)
   737	                rtf = total_time / audio_duration if audio_duration > 0 else 0
   738
   739	                log_request_completed(
   740	                    request_id=request_id,
   741	                    status_code=200,
   742	                    total_duration_ms=int(total_time * 1000),
   743	                    audio_duration_seconds=audio_duration,
   744	                    rtf=rtf,
   745	                    chunks_sent=chunks_count,
   746	                )
   747	                metrics_request_completed(
   748	                    status_code=200,
   749	                    duration_seconds=total_time,
   750	                    speaker=speaker,
   751	                    stream=True,
   752	                )
   753	                record_rtf(rtf=rtf, speaker=speaker)
   754	                record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
   755	                record_chunks_sent(chunks=chunks_count, speaker=speaker)
   756	                clear_request_context()
   757	            except Exception as e:
   758	                logger.exception(f"Streaming error: {e}")
   759	                # Log streaming failure
   760	                log_request_failed(


**Patching request metrics handling**


Input:
*** Begin Patch
*** Update File: veena3modal/api/fastapi_app.py
@@
         # Format encoding will be applied after generation (non-streaming only)
         # For streaming, only WAV is supported
         target_format = req.format
-
-        try:
-            # Get output sample rate - use_enum_values=True means req.output is already a string
-            output_sr = req.output if hasattr(req, 'output') else "16khz"
-
-            # Generate speech (non-streaming)
-            if req.chunking:
-                audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
-                    text=text,
-                    speaker=speaker,
-                    temperature=req.temperature,
-                    top_k=req.top_k,
-                    top_p=req.top_p,
-                    max_tokens=req.max_tokens,
-                    repetition_penalty=req.repetition_penalty,
-                    seed=req.seed,
-                    sample_rate=req.sample_rate,
-                    output_sample_rate=output_sr,
-                )
-            else:
-                audio_bytes, metrics = await tts_runtime.generate_speech(
-                    text=text,
-                    speaker=speaker,
-                    temperature=req.temperature,
-                    top_k=req.top_k,
-                    top_p=req.top_p,
-                    max_tokens=req.max_tokens,
-                    repetition_penalty=req.repetition_penalty,
-                    seed=req.seed,
-                    output_sample_rate=output_sr,
-                )
-
-            if audio_bytes is None:
-                return JSONResponse(
-                    status_code=500,
-                    content={
-                        "error": {
-                            "code": "GENERATION_FAILED",
-                            "message": "Audio generation failed. Check logs for details.",
-                            "request_id": request_id,
-                        }
-                    },
-                    headers={"X-Request-ID": request_id},
-                )
-
-            # Calculate RTF
-            total_time = time.time() - start_time
-            audio_duration = metrics.get("audio_duration_seconds", 0)
-            rtf = total_time / audio_duration if audio_duration > 0 else 0
-
-            # Build response headers
-            runtime = tts_runtime.get_runtime()
-            model_version = runtime.model_version if runtime else get_model_version()
-
-            # Use actual output sample rate from metrics
-            output_sample_rate = metrics.get("output_sample_rate", req.sample_rate)
-
-            headers = {
-                "X-Request-ID": request_id,
-                "X-Model-Version": model_version,
-                "X-Format": "wav",
-                "X-Sample-Rate": str(output_sample_rate),
-                "X-Stream": "false",
-                "X-Audio-Bytes": str(len(audio_bytes)),
-                "X-Audio-Seconds": f"{audio_duration:.2f}",
-                "X-TTFB-ms": str(metrics.get("ttfb_ms", 0)),
-                "X-RTF": f"{rtf:.3f}",
-                "X-Text-Chunked": str(metrics.get("text_chunked", False)).lower(),
-                "X-SR-Applied": str(metrics.get("sr_applied", False)).lower(),
-            }
-
-            if req.seed is not None:
-                headers["X-Seed"] = str(req.seed)
-
-            # Fire-and-forget sentence storage (non-blocking, after generation)
-            sentence_store = _get_sentence_store()
-            sentence_store.store_fire_and_forget(
-                request_id=request_id,
-                text=text,
-                speaker=speaker,
-                stream=False,
-                format=req.format,
-                temperature=req.temperature,
-                top_k=req.top_k,
-                top_p=req.top_p,
-                max_tokens=req.max_tokens,
-                repetition_penalty=req.repetition_penalty,
-                seed=req.seed,
-                text_chunked=metrics.get("text_chunked", False),
-                ttfb_ms=metrics.get("ttfb_ms"),
-                audio_duration_seconds=audio_duration,
-            )
-
-            # Log successful completion (lifecycle event + metrics)
-            total_duration_ms = int(total_time * 1000)
-            log_request_completed(
-                request_id=request_id,
-                status_code=200,
-                total_duration_ms=total_duration_ms,
-                audio_duration_seconds=audio_duration,
-                rtf=rtf,
-                chunks_sent=1,
-            )
-            metrics_request_completed(
-                status_code=200,
-                duration_seconds=total_time,
-                speaker=speaker,
-                stream=False,
-            )
-            record_ttfb(ttfb_seconds=metrics.get("ttfb_ms", 0) / 1000, speaker=speaker, stream=False)
-            record_rtf(rtf=rtf, speaker=speaker)
-            record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
-
-            clear_request_context()
-
-            # Apply format encoding if not WAV
-            media_type = "audio/wav"
-            if target_format != AudioFormat.WAV.value:
-                try:
-                    from veena3modal.audio.encoder import (
-                        OpusEncoder, MP3Encoder, MuLawEncoder, FLACEncoder
-                    )
-
-                    # Get sample rate from output (after potential SR)
-                    sample_rate = metrics.get("output_sample_rate", 16000)
-
-                    # Strip WAV header to get PCM data
-                    pcm_data = audio_bytes[44:] if len(audio_bytes) > 44 else audio_bytes
-
-                    if target_format == AudioFormat.OPUS.value:
-                        encoder = OpusEncoder(sample_rate=sample_rate)
-                        audio_bytes = encoder.encode(pcm_data, sample_rate)
-                        media_type = "audio/opus"
-                    elif target_format == AudioFormat.MP3.value:
-                        encoder = MP3Encoder(sample_rate=sample_rate)
-                        audio_bytes = encoder.encode(pcm_data, sample_rate)
-                        media_type = "audio/mpeg"
-                    elif target_format == AudioFormat.MULAW.value:
-                        encoder = MuLawEncoder()
-                        audio_bytes = encoder.encode(pcm_data, sample_rate)
-                        media_type = "audio/x-wav"
-                    elif target_format == AudioFormat.FLAC.value:
-                        encoder = FLACEncoder(sample_rate=sample_rate)
-                        audio_bytes = encoder.encode(pcm_data, sample_rate)
-                        media_type = "audio/flac"
-
-                    headers["X-Audio-Bytes"] = str(len(audio_bytes))
-                    logger.info(f"Encoded audio to {target_format}: {len(audio_bytes)} bytes")
-                except ImportError as e:
-                    logger.warning(f"Audio encoder not available: {e}")
-                    # Fall back to WAV
-                except Exception as e:
-                    logger.error(f"Format encoding failed: {e}")
-                    # Fall back to WAV
-
-            return Response(
-                content=audio_bytes,
-                media_type=media_type,
-                headers=headers,
-            )
-
-        except Exception as e:
-            logger.exception(f"TTS generation error: {e}")
-
-            # Log failure (lifecycle event + metrics)
-            log_request_failed(
-                request_id=request_id,
-                status_code=500,
-                error_code="INTERNAL_ERROR",
-                error_message=str(e),
-            )
-            metrics_request_failed(
-                status_code=500,
-                error_code="INTERNAL_ERROR",
-                speaker=speaker,
-            )
-            clear_request_context()
-
-            return JSONResponse(
-                status_code=500,
-                content={
-                    "error": {
-                        "code": "INTERNAL_ERROR",
-                        "message": str(e),
-                        "request_id": request_id,
-                    }
-                },
-                headers={"X-Request-ID": request_id},
-            )
+        inflight_current = _inflight_enter()
+        try:
+            try:
+                # Get output sample rate - use_enum_values=True means req.output is already a string
+                output_sr = req.output if hasattr(req, 'output') else "16khz"
+
+                # Generate speech (non-streaming)
+                if req.chunking:
+                    audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
+                        text=text,
+                        speaker=speaker,
+                        temperature=req.temperature,
+                        top_k=req.top_k,
+                        top_p=req.top_p,
+                        max_tokens=req.max_tokens,
+                        repetition_penalty=req.repetition_penalty,
+                        seed=req.seed,
+                        sample_rate=req.sample_rate,
+                        output_sample_rate=output_sr,
+                    )
+                else:
+                    audio_bytes, metrics = await tts_runtime.generate_speech(
+                        text=text,
+                        speaker=speaker,
+                        temperature=req.temperature,
+                        top_k=req.top_k,
+                        top_p=req.top_p,
+                        max_tokens=req.max_tokens,
+                        repetition_penalty=req.repetition_penalty,
+                        seed=req.seed,
+                        output_sample_rate=output_sr,
+                    )
+
+                if metrics is None:
+                    metrics = {}
+                metrics["request_inflight"] = max(
+                    int(metrics.get("request_inflight", 0) or 0),
+                    inflight_current,
+                )
+
+                if audio_bytes is None:
+                    return JSONResponse(
+                        status_code=500,
+                        content={
+                            "error": {
+                                "code": "GENERATION_FAILED",
+                                "message": "Audio generation failed. Check logs for details.",
+                                "request_id": request_id,
+                            }
+                        },
+                        headers={"X-Request-ID": request_id},
+                    )
+
+                # Calculate RTF
+                total_time = time.time() - start_time
+                audio_duration = metrics.get("audio_duration_seconds", 0)
+                rtf = total_time / audio_duration if audio_duration > 0 else 0
+
+                # Build response headers
+                runtime = tts_runtime.get_runtime()
+                model_version = runtime.model_version if runtime else get_model_version()
+
+                # Use actual output sample rate from metrics
+                output_sample_rate = metrics.get("output_sample_rate", req.sample_rate)
+
+                headers = {
+                    "X-Request-ID": request_id,
+                    "X-Model-Version": model_version,
+                    "X-Format": "wav",
+                    "X-Sample-Rate": str(output_sample_rate),
+                    "X-Stream": "false",
+                    "X-Audio-Bytes": str(len(audio_bytes)),
+                    "X-Audio-Seconds": f"{audio_duration:.2f}",
+                    "X-TTFB-ms": str(metrics.get("ttfb_ms", 0)),
+                    "X-RTF": f"{rtf:.3f}",
+                    "X-Text-Chunked": str(metrics.get("text_chunked", False)).lower(),
+                    "X-SR-Applied": str(metrics.get("sr_applied", False)).lower(),
+                }
+                headers.update(_build_perf_headers(metrics, request_inflight=inflight_current))
+
+                if req.seed is not None:
+                    headers["X-Seed"] = str(req.seed)
+
+                # Fire-and-forget sentence storage (non-blocking, after generation)
+                sentence_store = _get_sentence_store()
+                sentence_store.store_fire_and_forget(
+                    request_id=request_id,
+                    text=text,
+                    speaker=speaker,
+                    stream=False,
+                    format=req.format,
+                    temperature=req.temperature,
+                    top_k=req.top_k,
+                    top_p=req.top_p,
+                    max_tokens=req.max_tokens,
+                    repetition_penalty=req.repetition_penalty,
+                    seed=req.seed,
+                    text_chunked=metrics.get("text_chunked", False),
+                    ttfb_ms=metrics.get("ttfb_ms"),
+                    audio_duration_seconds=audio_duration,
+                )
+
+                # Log successful completion (lifecycle event + metrics)
+                total_duration_ms = int(total_time * 1000)
+                log_request_completed(
+                    request_id=request_id,
+                    status_code=200,
+                    total_duration_ms=total_duration_ms,
+                    audio_duration_seconds=audio_duration,
+                    rtf=rtf,
+                    chunks_sent=1,
+                )
+                metrics_request_completed(
+                    status_code=200,
+                    duration_seconds=total_time,
+                    speaker=speaker,
+                    stream=False,
+                )
+                record_ttfb(ttfb_seconds=metrics.get("ttfb_ms", 0) / 1000, speaker=speaker, stream=False)
+                record_rtf(rtf=rtf, speaker=speaker)
+                record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
+
+                # High-cardinality details are kept in logs; summarize key timings.
+                logger.info(
+                    "request_perf_summary",
+                    extra={
+                        "request_id": request_id,
+                        "speaker": speaker,
+                        "generation_ms": metrics.get("generation_ms"),
+                        "llm_token_total": metrics.get("llm_token_total"),
+                        "llm_batch_count": metrics.get("llm_batch_count"),
+                        "llm_batch_wall_ms_total": metrics.get("llm_batch_wall_ms_total"),
+                        "llm_batch_gpu_ms_total": metrics.get("llm_batch_gpu_ms_total"),
+                        "bicodec_decode_wall_ms": metrics.get("bicodec_decode_wall_ms"),
+                        "bicodec_decode_gpu_ms": metrics.get("bicodec_decode_gpu_ms"),
+                        "request_inflight": metrics.get("request_inflight"),
+                    },
+                )
+
+                clear_request_context()
+
+                # Apply format encoding if not WAV
+                media_type = "audio/wav"
+                if target_format != AudioFormat.WAV.value:
+                    try:
+                        from veena3modal.audio.encoder import (
+                            OpusEncoder, MP3Encoder, MuLawEncoder, FLACEncoder
+                        )
+
+                        # Get sample rate from output (after potential SR)
+                        sample_rate = metrics.get("output_sample_rate", 16000)
+
+                        # Strip WAV header to get PCM data
+                        pcm_data = audio_bytes[44:] if len(audio_bytes) > 44 else audio_bytes
+
+                        if target_format == AudioFormat.OPUS.value:
+                            encoder = OpusEncoder(sample_rate=sample_rate)
+                            audio_bytes = encoder.encode(pcm_data, sample_rate)
+                            media_type = "audio/opus"
+                        elif target_format == AudioFormat.MP3.value:
+                            encoder = MP3Encoder(sample_rate=sample_rate)
+                            audio_bytes = encoder.encode(pcm_data, sample_rate)
+                            media_type = "audio/mpeg"
+                        elif target_format == AudioFormat.MULAW.value:
+                            encoder = MuLawEncoder()
+                            audio_bytes = encoder.encode(pcm_data, sample_rate)
+                            media_type = "audio/x-wav"
+                        elif target_format == AudioFormat.FLAC.value:
+                            encoder = FLACEncoder(sample_rate=sample_rate)
+                            audio_bytes = encoder.encode(pcm_data, sample_rate)
+                            media_type = "audio/flac"
+
+                        headers["X-Audio-Bytes"] = str(len(audio_bytes))
+                        logger.info(f"Encoded audio to {target_format}: {len(audio_bytes)} bytes")
+                    except ImportError as e:
+                        logger.warning(f"Audio encoder not available: {e}")
+                        # Fall back to WAV
+                    except Exception as e:
+                        logger.error(f"Format encoding failed: {e}")
+                        # Fall back to WAV
+
+                return Response(
+                    content=audio_bytes,
+                    media_type=media_type,
+                    headers=headers,
+                )
+
+            except Exception as e:
+                logger.exception(f"TTS generation error: {e}")
+
+                # Log failure (lifecycle event + metrics)
+                log_request_failed(
+                    request_id=request_id,
+                    status_code=500,
+                    error_code="INTERNAL_ERROR",
+                    error_message=str(e),
+                )
+                metrics_request_failed(
+                    status_code=500,
+                    error_code="INTERNAL_ERROR",
+                    speaker=speaker,
+                )
+                clear_request_context()
+
+                return JSONResponse(
+                    status_code=500,
+                    content={
+                        "error": {
+                            "code": "INTERNAL_ERROR",
+                            "message": str(e),
+                            "request_id": request_id,
+                        }
+                    },
+                    headers={"X-Request-ID": request_id},
+                )
+        finally:
+            _inflight_exit()
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/api/fastapi_app.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Planning syntax error check**


I’ve wired API-level perf headers and in-flight tracking. I’m now running a syntax/type sanity pass on the modified modules before restarting and benchmarking.


python3 -m py_compile veena3modal/core/pipeline.py veena3modal/services/tts_runtime.py veena3modal/api/fastapi_app.py


python3 -m py_compile scripts/stress_test_optimized_local.py


Output:
Chunk ID: 28080e
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 0
Output:


Output:
Chunk ID: b16631
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 0
Output:


**Fixing token timing and adding bicodec metrics**