Output:
Chunk ID: b935d9
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 2035
Output:
        normalizer_func = None
        if req.normalize:
            try:
                from veena3modal.processing.text_normalizer import normalize_text
                normalizer_func = lambda t: normalize_text(t, verbose=req.normalize_verbose)
            except ImportError:
                logger.warning("TextNormalizer not available, skipping text normalization")

        text = req.get_normalized_text(normalizer_func=normalizer_func)

        # Set request context for structured logging
        set_request_context(request_id=request_id)

        # Log request received (lifecycle event + metrics)
        log_request_received(
            request_id=request_id,
            text_length=len(text),
            speaker=speaker,
            stream=req.stream,
            format=req.format,
        )
        metrics_request_received(speaker=speaker, stream=req.stream, format=req.format)

        # Streaming mode (M4)
        if req.stream:
            return await _handle_streaming_request(req, request_id, text, speaker, start_time)

        # Format encoding will be applied after generation (non-streaming only)
        # For streaming, only WAV is supported
        target_format = req.format

        try:
            # Get output sample rate - use_enum_values=True means req.output is already a string
            output_sr = req.output if hasattr(req, 'output') else "16khz"

            # Generate speech (non-streaming)
            if req.chunking:
                audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
                    text=text,
                    speaker=speaker,
                    temperature=req.temperature,
                    top_k=req.top_k,
                    top_p=req.top_p,
                    max_tokens=req.max_tokens,
                    repetition_penalty=req.repetition_penalty,
                    seed=req.seed,
                    sample_rate=req.sample_rate,
                    output_sample_rate=output_sr,
                )
            else:
                audio_bytes, metrics = await tts_runtime.generate_speech(
                    text=text,
                    speaker=speaker,
                    temperature=req.temperature,
                    top_k=req.top_k,
                    top_p=req.top_p,
                    max_tokens=req.max_tokens,
                    repetition_penalty=req.repetition_penalty,
                    seed=req.seed,
                    output_sample_rate=output_sr,
                )

            if audio_bytes is None:
                return JSONResponse(
                    status_code=500,
                    content={
                        "error": {
                            "code": "GENERATION_FAILED",
                            "message": "Audio generation failed. Check logs for details.",
                            "request_id": request_id,
                        }
                    },
                    headers={"X-Request-ID": request_id},
                )

            # Calculate RTF
            total_time = time.time() - start_time
            audio_duration = metrics.get("audio_duration_seconds", 0)
            rtf = total_time / audio_duration if audio_duration > 0 else 0

            # Build response headers
            runtime = tts_runtime.get_runtime()
            model_version = runtime.model_version if runtime else get_model_version()

            # Use actual output sample rate from metrics
            output_sample_rate = metrics.get("output_sample_rate", req.sample_rate)

            headers = {
                "X-Request-ID": request_id,
                "X-Model-Version": model_version,
                "X-Format": "wav",
                "X-Sample-Rate": str(output_sample_rate),
                "X-Stream": "false",
                "X-Audio-Bytes": str(len(audio_bytes)),
                "X-Audio-Seconds": f"{audio_duration:.2f}",
                "X-TTFB-ms": str(metrics.get("ttfb_ms", 0)),
                "X-RTF": f"{rtf:.3f}",
                "X-Text-Chunked": str(metrics.get("text_chunked", False)).lower(),
                "X-SR-Applied": str(metrics.get("sr_applied", False)).lower(),
            }

            if req.seed is not None:
                headers["X-Seed"] = str(req.seed)

            # Fire-and-forget sentence storage (non-blocking, after generation)
            sentence_store = _get_sentence_store()
            sentence_store.store_fire_and_forget(
                request_id=request_id,
                text=text,
                speaker=speaker,
                stream=False,
                format=req.format,
                temperature=req.temperature,
                top_k=req.top_k,
                top_p=req.top_p,
                max_tokens=req.max_tokens,
                repetition_penalty=req.repetition_penalty,
                seed=req.seed,
                text_chunked=metrics.get("text_chunked", False),
                ttfb_ms=metrics.get("ttfb_ms"),
                audio_duration_seconds=audio_duration,
            )

            # Log successful completion (lifecycle event + metrics)
            total_duration_ms = int(total_time * 1000)
            log_request_completed(
                request_id=request_id,
                status_code=200,
                total_duration_ms=total_duration_ms,
                audio_duration_seconds=audio_duration,
                rtf=rtf,
                chunks_sent=1,
            )
            metrics_request_completed(
                status_code=200,
                duration_seconds=total_time,
                speaker=speaker,
                stream=False,
            )
            record_ttfb(ttfb_seconds=metrics.get("ttfb_ms", 0) / 1000, speaker=speaker, stream=False)
            record_rtf(rtf=rtf, speaker=speaker)
            record_audio_duration(duration_seconds=audio_duration, speaker=speaker)

            clear_request_context()

            # Apply format encoding if not WAV
            media_type = "audio/wav"
            if target_format != AudioFormat.WAV.value:
                try:
                    from veena3modal.audio.encoder import (
                        OpusEncoder, MP3Encoder, MuLawEncoder, FLACEncoder
                    )

                    # Get sample rate from output (after potential SR)
                    sample_rate = metrics.get("output_sample_rate", 16000)

                    # Strip WAV header to get PCM data
                    pcm_data = audio_bytes[44:] if len(audio_bytes) > 44 else audio_bytes

                    if target_format == AudioFormat.OPUS.value:
                        encoder = OpusEncoder(sample_rate=sample_rate)
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/opus"
                    elif target_format == AudioFormat.MP3.value:
                        encoder = MP3Encoder(sample_rate=sample_rate)
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/mpeg"
                    elif target_format == AudioFormat.MULAW.value:
                        encoder = MuLawEncoder()
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/x-wav"
                    elif target_format == AudioFormat.FLAC.value:
                        encoder = FLACEncoder(sample_rate=sample_rate)
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/flac"

                    headers["X-Audio-Bytes"] = str(len(audio_bytes))
                    logger.info(f"Encoded audio to {target_format}: {len(audio_bytes)} bytes")
                except ImportError as e:
                    logger.warning(f"Audio encoder not available: {e}")
                    # Fall back to WAV


Output:
Chunk ID: 1e85cf
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2468
Output:
                        encoder = OpusEncoder(sample_rate=sample_rate)
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/opus"
                    elif target_format == AudioFormat.MP3.value:
                        encoder = MP3Encoder(sample_rate=sample_rate)
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/mpeg"
                    elif target_format == AudioFormat.MULAW.value:
                        encoder = MuLawEncoder()
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/x-wav"
                    elif target_format == AudioFormat.FLAC.value:
                        encoder = FLACEncoder(sample_rate=sample_rate)
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/flac"

                    headers["X-Audio-Bytes"] = str(len(audio_bytes))
                    logger.info(f"Encoded audio to {target_format}: {len(audio_bytes)} bytes")
                except ImportError as e:
                    logger.warning(f"Audio encoder not available: {e}")
                    # Fall back to WAV
                except Exception as e:
                    logger.error(f"Format encoding failed: {e}")
                    # Fall back to WAV

            return Response(
                content=audio_bytes,
                media_type=media_type,
                headers=headers,
            )

        except Exception as e:
            logger.exception(f"TTS generation error: {e}")

            # Log failure (lifecycle event + metrics)
            log_request_failed(
                request_id=request_id,
                status_code=500,
                error_code="INTERNAL_ERROR",
                error_message=str(e),
            )
            metrics_request_failed(
                status_code=500,
                error_code="INTERNAL_ERROR",
                speaker=speaker,
            )
            clear_request_context()

            return JSONResponse(
                status_code=500,
                content={
                    "error": {
                        "code": "INTERNAL_ERROR",
                        "message": str(e),
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

    async def _handle_streaming_request(req, request_id: str, text: str, speaker: str, start_time: float):
        """
        Handle streaming TTS request.

        Returns a StreamingResponse that yields WAV header + PCM chunks.
        True streaming: first bytes sent ASAP before full audio is generated.
        """
        from fastapi.responses import StreamingResponse
        from veena3modal.api.schemas import AudioFormat
        from veena3modal.services import tts_runtime

        # Check if streaming pipeline is available
        runtime = tts_runtime.get_runtime()
        if runtime is None or runtime.streaming_pipeline is None:
            return JSONResponse(
                status_code=503,
                content={
                    "error": {
                        "code": "STREAMING_UNAVAILABLE",
                        "message": "Streaming pipeline not initialized.",
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

        # Non-WAV formats not implemented yet (M5)
        if req.format != AudioFormat.WAV.value:
            return JSONResponse(
                status_code=501,
                content={
                    "error": {
                        "code": "FORMAT_NOT_IMPLEMENTED",
                        "message": f"Streaming format '{req.format}' not yet implemented. Use format=wav.",
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

        # Set request context for structured logging
        set_request_context(request_id=request_id)

        # Create async generator for streaming response
        async def audio_stream_generator():
            """Yields audio chunks as they're generated."""
            final_metrics = {}
            first_chunk_sent = False
            chunks_count = 0
            try:
                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
                    text=text,
                    speaker=speaker,
                    temperature=req.temperature,
                    top_k=req.top_k,
                    top_p=req.top_p,
                    max_tokens=req.max_tokens,
                    repetition_penalty=req.repetition_penalty,
                    seed=req.seed,
                    enable_chunking=req.chunking,
                ):
                    final_metrics = metrics
                    chunks_count += 1

                    # Fire-and-forget sentence storage after first chunk (don't block TTFB)
                    if not first_chunk_sent:
                        first_chunk_sent = True
                        ttfb_ms = metrics.get("ttfb_ms", 0)

                        # Log first audio emitted (TTFB marker)
                        log_first_audio_emitted(
                            request_id=request_id,
                            ttfb_ms=ttfb_ms,
                            chunk_size_bytes=len(audio_chunk),
                        )
                        record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)

                        sentence_store = _get_sentence_store()
                        sentence_store.store_fire_and_forget(
                            request_id=request_id,
                            text=text,
                            speaker=speaker,
                            stream=True,
                            format=req.format,
                            temperature=req.temperature,
                            top_k=req.top_k,
                            top_p=req.top_p,
                            max_tokens=req.max_tokens,
                            repetition_penalty=req.repetition_penalty,
                            seed=req.seed,
                            text_chunked=metrics.get("text_chunked", False),
                            ttfb_ms=ttfb_ms,
                            # audio_duration_seconds not known yet for streaming
                        )

                    yield audio_chunk

                # Stream completed - log metrics
                total_time = time.time() - start_time
                audio_duration = final_metrics.get("audio_duration_seconds", 0)
                rtf = total_time / audio_duration if audio_duration > 0 else 0

                log_request_completed(
                    request_id=request_id,
                    status_code=200,
                    total_duration_ms=int(total_time * 1000),
                    audio_duration_seconds=audio_duration,
                    rtf=rtf,
                    chunks_sent=chunks_count,
                )
                metrics_request_completed(
                    status_code=200,
                    duration_seconds=total_time,
                    speaker=speaker,
                    stream=True,
                )
                record_rtf(rtf=rtf, speaker=speaker)
                record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
                record_chunks_sent(chunks=chunks_count, speaker=speaker)
                clear_request_context()
            except Exception as e:
                logger.exception(f"Streaming error: {e}")
                # Log streaming failure
                log_request_failed(
                    request_id=request_id,
                    status_code=500,
                    error_code="STREAMING_ERROR",
                    error_message=str(e),
                )
                metrics_request_failed(
                    status_code=500,
                    error_code="STREAMING_ERROR",
                    speaker=speaker,
                )
                clear_request_context()
                # Can't return error once streaming started
                # Best we can do is stop yielding
                return

        # Build response headers
        # Note: Some headers can't be set dynamically during streaming
        # TTFB and RTF will be logged but not in headers for streaming
        model_version = runtime.model_version if runtime else get_model_version()

        headers = {
            "X-Request-ID": request_id,
            "X-Model-Version": model_version,
            "X-Format": "wav",
            "X-Sample-Rate": str(req.sample_rate),
            "X-Stream": "true",
            "X-Chunking-Enabled": str(req.chunking).lower(),
            # Transfer-Encoding: chunked is set automatically by StreamingResponse
        }

        if req.seed is not None:
            headers["X-Seed"] = str(req.seed)

        return StreamingResponse(
            audio_stream_generator(),
            media_type="audio/wav",
            headers=headers,
        )

    # === WebSocket Support (M5b) ===
    try:
        from veena3modal.api.websocket_handler import add_websocket_routes
        add_websocket_routes(app)
        logger.info("WebSocket support enabled at /v1/tts/ws")
    except ImportError as e:
        logger.warning(f"WebSocket support not available: {e}")

    return app


Output:
Chunk ID: 14f2ca
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1290
Output:
                        elif data.get("event") == "ping":
                            await websocket.send_text(json.dumps({"event": "pong"}))
                    except asyncio.TimeoutError:
                        continue
                    except Exception:
                        break
            except Exception:
                pass

        cancel_task = asyncio.create_task(listen_for_cancel())

        try:
            # Apply text normalization if enabled
            text = req.text
            if req.normalize:
                try:
                    from veena3modal.processing.text_normalizer import normalize_text
                    text = normalize_text(text)
                except ImportError:
                    pass  # Normalizer not available

            # Stream audio chunks
            async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
                text=text,
                speaker=speaker,
                temperature=req.temperature,
                top_k=req.top_k,
                top_p=req.top_p,
                max_tokens=req.max_tokens,
                repetition_penalty=req.repetition_penalty,
                seed=req.seed,
                enable_chunking=req.chunking,
            ):
                if cancelled:
                    logger.info(f"Generation cancelled: {request_id}")
                    break

                final_metrics = metrics
                chunks_sent += 1
                total_bytes += len(audio_chunk)

                # Record TTFB on first chunk
                if first_chunk_time is None:
                    first_chunk_time = time.time()
                    ttfb_ms = int((first_chunk_time - start_time) * 1000)
                    log_first_audio_emitted(
                        request_id=request_id,
                        ttfb_ms=ttfb_ms,
                        chunk_size_bytes=len(audio_chunk),
                    )
                    record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)

                # Send audio as binary frame
                await websocket.send_bytes(audio_chunk)

                # Send progress every 10 chunks
                if chunks_sent % 10 == 0:
                    await websocket.send_text(create_progress_message(
                        chunks_sent=chunks_sent,
                        bytes_sent=total_bytes,
                        elapsed_ms=int((time.time() - start_time) * 1000),
                    ))

        finally:
            cancel_task.cancel()
            try:
                await cancel_task
            except asyncio.CancelledError:
                pass

        # === SEND COMPLETION ===
        total_time = time.time() - start_time
        audio_duration = final_metrics.get("audio_duration_seconds", 0)
        rtf = total_time / audio_duration if audio_duration > 0 else 0

        completion_metrics = {
            "request_id": request_id,
            "chunks_sent": chunks_sent,
            "total_bytes": total_bytes,
            "audio_duration_seconds": round(audio_duration, 2),
            "total_time_seconds": round(total_time, 2),
            "ttfb_ms": int((first_chunk_time - start_time) * 1000) if first_chunk_time else 0,
            "rtf": round(rtf, 3),
            "cancelled": cancelled,
        }

        await websocket.send_text(create_complete_message(
            request_id=request_id,
            metrics=completion_metrics,
        ))

        # Log completion
        if not cancelled:
            log_request_completed(
                request_id=request_id,
                status_code=200,
                total_duration_ms=int(total_time * 1000),
                audio_duration_seconds=audio_duration,
                rtf=rtf,
                chunks_sent=chunks_sent,
            )
            metrics_request_completed(
                status_code=200,
                duration_seconds=total_time,
                speaker=speaker,
                stream=True,
            )
            record_rtf(rtf=rtf, speaker=speaker)
            record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
            record_chunks_sent(chunks=chunks_sent, speaker=speaker)

        # Close connection gracefully
        await websocket.close(code=1000)

    except Exception as e:
        logger.exception(f"WebSocket error: {e}")
        log_request_failed(
            request_id=request_id,
            status_code=500,
            error_code="WEBSOCKET_ERROR",
            error_message=str(e),
        )
        metrics_request_failed(
            status_code=500,
            error_code="WEBSOCKET_ERROR",
        )

        try:
            await websocket.send_text(create_error_message(
                code="INTERNAL_ERROR",
                message=str(e),
                request_id=request_id,
            ))
            await websocket.close(code=1011)
        except Exception:
            pass

    finally:
        clear_request_context()


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


rg -n "def generate_speech\(|def generate_speech_chunked\(|def generate_speech_streaming\(|def __init__|active_requests|request_inflight" /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py


Output:
Chunk ID: 5f2bbd
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 273
Output:
365:async def generate_speech(
418:    active_requests = _with_inflight_delta(1)
430:                "request_inflight": active_requests,
516:        active_requests = _with_inflight_delta(-1)
518:            metrics["request_inflight"] = active_requests
519:            metrics["active_requests"] = active_requests
529:                    "active_requests": active_requests,
600:async def generate_speech_chunked(
646:    active_requests = _with_inflight_delta(1)
657:                    "request_inflight": active_requests,
777:            metrics["active_requests"] = active_requests - 1
778:            metrics["request_inflight"] = active_requests - 1
786:        active_requests = _with_inflight_delta(-1)
787:        metrics["active_requests"] = active_requests
788:        metrics["request_inflight"] = active_requests
794:async def generate_speech_streaming(
870:    active_requests = _with_inflight_delta(1)
877:                "request_inflight": active_requests,
970:        active_requests = _with_inflight_delta(-1)
980:                    "request_inflight": active_requests,


Summary: auto


sed -n '340,460p' /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py


Output:
Chunk ID: 53b4a5
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 968
Output:
                else:
                    logger.warning(f"  Failed to capture globals for {speaker_name}")
            except Exception as e:
                logger.warning(f"  Error caching globals for {speaker_name}: {e}")

    # Run async warmup in event loop
    try:
        loop = asyncio.get_event_loop()
        if loop.is_running():
            # Already in async context (unlikely at startup, but handle it)
            import concurrent.futures
            with concurrent.futures.ThreadPoolExecutor() as pool:
                pool.submit(lambda: asyncio.run(_run_all())).result()
        else:
            loop.run_until_complete(_run_all())
    except RuntimeError:
        # No event loop exists yet
        asyncio.run(_run_all())

    elapsed = (time.time() - t_start) * 1000
    cached = len(runtime.speaker_global_cache)
    total = len(SPEAKER_MAP)
    logger.info(f"Speaker globals cached: {cached}/{total} speakers in {elapsed:.0f}ms")

async def generate_speech(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    output_sample_rate: str = "16khz",
    request_id: Optional[str] = None,
) -> Tuple[Optional[bytes], Dict[str, Any]]:
    """
    Generate speech audio (non-streaming).

    Args:
        text: Text to synthesize (already normalized)
        speaker: Internal speaker name (resolved)
        temperature: Sampling temperature
        top_k: Top-k sampling
        top_p: Nucleus sampling
        max_tokens: Maximum tokens to generate
        repetition_penalty: Repetition penalty
        seed: Random seed for reproducibility
        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)

    Returns:
        Tuple of (audio_bytes, metrics_dict)
        audio_bytes: WAV audio bytes (16kHz or 48kHz, 16-bit PCM) or None if failed
        metrics_dict: Dictionary with timing metrics

    Raises:
        RuntimeError: If runtime not initialized
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()
    metrics = {
        "request_id": request_id,
        "ttfb_ms": 0,
        "generation_ms": 0,
        "tokens_generated": 0,
        "audio_duration_seconds": 0.0,
        "sr_applied": False,
        "output_sample_rate": 16000,
    }

    if request_id is None:
        import uuid
        request_id = f"runtime-ns-{uuid.uuid4().hex[:10]}"
        metrics["request_id"] = request_id

    active_requests = _with_inflight_delta(1)
    start_time = time.time()

    pipeline_timings = {}

    try:
        logger.info(
            "tts_runtime_nonstream_start",
            extra={
                "speaker": speaker,
                "request_id": request_id,
                "text_length": len(text),
                "request_inflight": active_requests,
                "temperature": temperature,
                "top_k": top_k,
                "top_p": top_p,
                "max_tokens": max_tokens,
            },
        )

        gen_kwargs = dict(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
        )

        if _timing_enabled():
            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
                **gen_kwargs,
                return_timing=True,
                request_id=request_id,
            )
        else:
            audio_bytes = await runtime.pipeline.generate_speech_indic(
                **gen_kwargs,
                request_id=request_id,
            )


sed -n '590,760p' /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py


sed -n '760,980p' /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py


Output:
Chunk ID: 2fc8d7
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 1660
Output:
        byte_rate,
        block_align,
        bits_per_sample,
        b'data',
        data_size,
    )

    return wav_header + upsampled_int16.tobytes()

async def generate_speech_chunked(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    sample_rate: int = 16000,
    output_sample_rate: str = "16khz",
    request_id: Optional[str] = None,
) -> Tuple[Optional[bytes], Dict[str, Any]]:
    """
    Generate speech with automatic text chunking for long inputs.

    Uses LongTextProcessor to split text and stitch audio.

    Args:
        text: Text to synthesize
        speaker: Internal speaker name
        ... (same as generate_speech)
        sample_rate: Internal generation sample rate (always 16000)
        output_sample_rate: "16khz" or "48khz" (triggers super-resolution)

    Returns:
        Tuple of (audio_bytes, metrics_dict)
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()
    if request_id is None:
        import uuid
        request_id = f"runtime-chunk-{uuid.uuid4().hex[:10]}"
    metrics = {
        "request_id": request_id,
        "ttfb_ms": 0,
        "generation_ms": 0,
        "chunks_processed": 0,
        "audio_duration_seconds": 0.0,
        "text_chunked": False,
        "sr_applied": False,
        "output_sample_rate": 16000,
    }

    active_requests = _with_inflight_delta(1)
    start_time = time.time()

    try:
        if _timing_enabled():
            logger.info(
                "tts_runtime_chunking_start",
                extra={
                    "speaker": speaker,
                    "request_id": request_id,
                    "text_length": len(text),
                    "request_inflight": active_requests,
                    "chunking": True,
                },
            )
        # Import long text processor
        from veena3modal.processing.long_text_processor import LongTextProcessor

        long_processor = LongTextProcessor(pipeline=runtime.pipeline)

        # Check if chunking is needed
        if long_processor.should_chunk(text):
            metrics["text_chunked"] = True
            if _timing_enabled():
                chunk_result = await long_processor.generate_with_chunking(
                    text=text,
                    speaker=speaker,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    sample_rate=sample_rate,
                    request_id=request_id,
                    return_timing=True,
                )
                if isinstance(chunk_result, tuple):
                    audio_bytes, chunk_timing = chunk_result
                else:
                    audio_bytes = chunk_result
                    chunk_timing = {}
            else:
                audio_bytes = await long_processor.generate_with_chunking(
                    text=text,
                    speaker=speaker,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    sample_rate=sample_rate,
                    request_id=request_id,
                )
                chunk_timing = {}
        else:
            # Short text: direct generation
            gen_kwargs = dict(
                speaker=speaker,
                text=text,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
                request_id=request_id,
            )
            if _timing_enabled():
                audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
                    **gen_kwargs,
                    return_timing=True,
                    request_id=request_id,
                )
                chunk_timing = pipeline_timings
            else:
                audio_bytes = await runtime.pipeline.generate_speech_indic(
                    **gen_kwargs,
                    request_id=request_id,
                )
                chunk_timing = {}

        generation_time = time.time() - start_time
        metrics["generation_ms"] = int(generation_time * 1000)
        metrics["ttfb_ms"] = metrics["generation_ms"]

        if chunk_timing:
            metrics.update({
                "llm_token_total": chunk_timing.get("token_total", 0),
                "llm_batch_count": chunk_timing.get("batch_count", 0),
                "llm_batch_gpu_ms_total": chunk_timing.get("llm_gpu_batch_ms_total", 0),
                "llm_batch_wall_ms_total": chunk_timing.get("llm_wall_ms", metrics["generation_ms"]),
                "llm_parse_ms": chunk_timing.get("parse_ms", 0),
                "llm_decode_cpu_ms": chunk_timing.get("decode_cpu_ms", 0),
                "llm_batch_wall_ms_min": chunk_timing.get("llm_batch_wall_ms_min", 0),
                "llm_batch_wall_ms_max": chunk_timing.get("llm_batch_wall_ms_max", 0),
                "llm_batch_gpu_ms_min": chunk_timing.get("llm_batch_gpu_ms_min", 0),
                "llm_batch_gpu_ms_max": chunk_timing.get("llm_batch_gpu_ms_max", 0),
                "llm_batch_gpu_ms_p50": chunk_timing.get("llm_batch_gpu_ms_p50", 0),
                "tokens_per_batch_min": chunk_timing.get("tokens_per_batch_min"),
                "tokens_per_batch_max": chunk_timing.get("tokens_per_batch_max"),
                "tokens_per_batch_p50": chunk_timing.get("tokens_per_batch_p50"),
                "llm_batch_tokens": chunk_timing.get("token_batches", []),
            })
            if metrics["text_chunked"]:
                metrics["chunks_processed"] = chunk_timing.get("chunks_processed", 0)

        if audio_bytes:
            # Apply super-resolution if requested and available
            final_sample_rate = 16000
            if output_sample_rate == "48khz" and runtime.sr_service and runtime.sr_service.is_loaded:
                try:
                    sr_start = time.time()
                    logger.info(f"Applying super-resolution to chunked audio ({len(audio_bytes)} bytes)...")


Output:
Chunk ID: 6cf428
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2227
Output:
                    logger.info(f"Applying super-resolution to chunked audio ({len(audio_bytes)} bytes)...")
                    audio_bytes = _apply_super_resolution(audio_bytes, runtime.sr_service)
                    sr_time = (time.time() - sr_start) * 1000
                    metrics["sr_ms"] = int(sr_time)
                    metrics["sr_applied"] = True
                    final_sample_rate = 48000
                    logger.info(f"✅ Super-resolution applied in {sr_time:.1f}ms, output={len(audio_bytes)} bytes")
                except Exception as e:
                    logger.warning(f"Super-resolution failed, returning 16kHz: {e}")
                    import traceback
                    traceback.print_exc()

            metrics["output_sample_rate"] = final_sample_rate
            audio_duration = (len(audio_bytes) - 44) / (final_sample_rate * 2)
            metrics["audio_duration_seconds"] = max(0.0, audio_duration)

        if _timing_enabled():
            metrics["active_requests"] = active_requests - 1
            metrics["request_inflight"] = active_requests - 1
            logger.info("tts_runtime_chunking_complete", extra=metrics)
        return audio_bytes, metrics

    except Exception as e:
        logger.error(f"Chunked speech generation failed: {e}")
        raise
    finally:
        active_requests = _with_inflight_delta(-1)
        metrics["active_requests"] = active_requests
        metrics["request_inflight"] = active_requests

from typing import AsyncGenerator

async def generate_speech_streaming(
    text: str,
    speaker: str,
    temperature: float = 0.8,
    top_k: int = 50,
    top_p: float = 1.0,
    max_tokens: int = 4096,
    repetition_penalty: float = 1.05,
    seed: Optional[int] = None,
    enable_chunking: bool = True,
    request_id: Optional[str] = None,
) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
    """
    Generate speech audio with true streaming (yields chunks as they're generated).

    This is the core streaming implementation for M4.
    First yield includes WAV header + first PCM chunk.
    Subsequent yields are raw PCM chunks.

    Args:
        text: Text to synthesize (already normalized)
        speaker: Internal speaker name (resolved)
        temperature: Sampling temperature
        top_k: Top-k sampling
        top_p: Nucleus sampling
        max_tokens: Maximum tokens to generate
        repetition_penalty: Repetition penalty
        seed: Random seed for reproducibility
        enable_chunking: Enable text chunking for long inputs with voice consistency

    Yields:
        Tuple of (audio_bytes, metrics_dict)
        - First yield: WAV header (44 bytes) prepended to first PCM chunk
        - Subsequent yields: Raw PCM chunks (int16, 16kHz)
        - Final yield: metrics_dict has final timing info

    Raises:
        RuntimeError: If runtime not initialized or streaming pipeline unavailable
    """
    if not is_initialized():
        raise RuntimeError("TTS runtime not initialized")

    runtime = get_runtime()

    if request_id is None:
        import uuid
        request_id = f"runtime-stream-{uuid.uuid4().hex[:10]}"

    if runtime.streaming_pipeline is None:
        raise RuntimeError("Streaming pipeline not available")

    # Import audio utils for WAV header
    from veena3modal.audio.utils import create_wav_header

    # Metrics tracking
    metrics = {
        "request_id": request_id,
        "speaker": speaker,
        "ttfb_ms": 0,
        "chunks_sent": 0,
        "total_bytes": 0,
        "audio_duration_seconds": 0.0,
        "text_chunked": False,
        "text_length": len(text),
    }
    pipeline_metrics: Dict[str, Any] = {
        "request_id": request_id,
        "speaker": speaker,
        "text_length": len(text),
        "text_chunked": False,
    }

    start_time = time.time()
    first_chunk_time = None
    total_pcm_bytes = 0
    sample_rate = 16000  # BiCodec sample rate
    active_requests = _with_inflight_delta(1)
    if _timing_enabled():
        logger.info(
            "tts_runtime_stream_start",
            extra={
                "speaker": speaker,
                "text_length": len(text),
                "request_inflight": active_requests,
                "chunking": enable_chunking,
                "request_id": request_id,
            },
        )

    try:
        # Check if we need text chunking
        from veena3modal.processing.long_text_processor import LongTextProcessor
        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
        needs_chunking = enable_chunking and long_processor.should_chunk(text)

        if needs_chunking:
            # Chunked streaming with voice consistency (global token caching)
            metrics["text_chunked"] = True
            async for audio_chunk, chunk_metrics in _stream_chunked_text(
                runtime,
                long_processor,
                text,
                speaker,
                temperature,
                top_k,
                top_p,
                max_tokens,
                repetition_penalty,
                seed,
                sample_rate,
                request_id,
                request_metrics=pipeline_metrics,
            ):
                if first_chunk_time is None:
                    first_chunk_time = time.time()
                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                    # Create streaming WAV header (size=0 for unknown length)
                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
                    audio_chunk = wav_header + audio_chunk

                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
                metrics["chunks_sent"] += 1
                metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
                if chunk_metrics:
                    metrics.update(chunk_metrics)

                yield audio_chunk, metrics
        else:
            # Simple streaming (no text chunking)
            # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
            # use continuation mode to skip the ~110ms global token pre-roll.
            # The model jumps straight to semantic token generation.
            cached_globals = runtime.speaker_global_cache.get(speaker)

            if cached_globals:
                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                    speaker=speaker,
                    text=text,
                    global_ids=cached_globals,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    request_id=request_id,
                    request_metrics=pipeline_metrics,
                )
            else:
                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
                    speaker=speaker,
                    text=text,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    request_id=request_id,
                    request_metrics=pipeline_metrics,
                )

            async for audio_chunk in stream_gen:
                if first_chunk_time is None:
                    first_chunk_time = time.time()
                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
                    audio_chunk = wav_header + audio_chunk

                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
                metrics["chunks_sent"] += 1
                metrics["total_bytes"] = total_pcm_bytes + 44
                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
                yield audio_chunk, metrics
    finally:
        active_requests = _with_inflight_delta(-1)
        metrics.update(pipeline_metrics)
        metrics["generation_ms"] = int((time.time() - start_time) * 1000)
        metrics["text_chunked"] = bool(metrics.get("text_chunked", False))
        if _timing_enabled():
            logger.info(
                "tts_runtime_stream_end",
                extra={
                    "speaker": speaker,
                    "request_id": request_id,
                    "request_inflight": active_requests,


sed -n '980,1060p' /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py


Output:
Chunk ID: bd0283
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 773
Output:
                    "request_inflight": active_requests,
                    "chunks_sent": metrics.get("chunks_sent", 0),
                    "generation_ms": metrics["generation_ms"],
                },
            )

async def _stream_chunked_text(
    runtime: TTSRuntime,
    long_processor,
    text: str,
    speaker: str,
    temperature: float,
    top_k: int,
    top_p: float,
    max_tokens: int,
    repetition_penalty: float,
    seed: Optional[int],
    sample_rate: int,
    request_id: Optional[str] = None,
    request_metrics: Optional[Dict[str, Any]] = None,
) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
    """
    Internal helper: stream chunked text with voice consistency.

    Uses global token caching from first chunk to maintain voice across chunks.
    """
    from veena3modal.audio.crossfade import crossfade_bytes_int16

    # Chunk text using the long text processor's chunker
    chunks = long_processor.chunk_text(text)

    if not chunks:
        return

    captured_globals: Optional[List[int]] = None
    previous_chunk_tail: Optional[bytes] = None
    chunk_metrics = request_metrics if request_metrics is not None else {"request_id": request_id}
    if "request_id" not in chunk_metrics:
        chunk_metrics["request_id"] = request_id
    if "chunks_processed" not in chunk_metrics:
        chunk_metrics["chunks_processed"] = 0

    for i, chunk_text in enumerate(chunks):
        chunk_metrics["chunks_processed"] = i + 1

        if i == 0:
            # First chunk: capture global tokens for voice consistency
            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
                speaker=speaker,
                text=chunk_text,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
                request_id=request_id,
                request_metrics=chunk_metrics,
            ):
                if captured_globals is None and global_ids:
                    captured_globals = global_ids

                # Crossfade with previous tail (inter-chunk stitching)
                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                    previous_chunk_tail,
                    audio_bytes,
                    sample_rate_hz=sample_rate,
                    crossfade_ms=50,
                )

                if to_emit:
                    yield to_emit, chunk_metrics
        else:
            # Continuation chunks: use captured globals for voice consistency
            if captured_globals is None:
                logger.warning(f"No captured globals for chunk {i+1}, using regular streaming")
                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
                    speaker=speaker,
                    text=chunk_text,
                    temperature=temperature,


sed -n '1060,1135p' /home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py


Output:
Chunk ID: 0a3686
Wall time: 0.0518 seconds
Process exited with code 0
Original token count: 463
Output:
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    request_id=request_id,
                    request_metrics=chunk_metrics,
                ):
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        audio_bytes,
                        sample_rate_hz=sample_rate,
                        crossfade_ms=50,
                    )
                    if to_emit:
                        yield to_emit, chunk_metrics
            else:
                # Use continuation method with cached globals
                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                    speaker=speaker,
                    text=chunk_text,
                    global_ids=captured_globals,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    request_id=request_id,
                    request_metrics=chunk_metrics,
                ):
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        audio_bytes,
                        sample_rate_hz=sample_rate,
                        crossfade_ms=50,
                    )
                    if to_emit:
                        yield to_emit, chunk_metrics

    # Flush remaining tail
    if previous_chunk_tail:
        yield previous_chunk_tail, chunk_metrics


I’ve confirmed the timing hooks are present through `tts_runtime` and already accept `request_id`, so I’ll now (1) propagate IDs into FastAPI/WebSocket logs, (2) return per-request timing in headers/event payload, and (3) add a dedicated local stress script focused on concurrency, per-request token/decode breakdowns, and GPU timing capture.


sed -n '1,260p' /home/ubuntu/veenaModal/scripts/stress_test_local.py


Output:
Chunk ID: afcb1d
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 2059
Output:
#!/usr/bin/env python3
"""
Local Stress Test for Veena3 TTS Server.

Progressive load ramp with GPU monitoring, latency percentiles,
error categorization, and throughput measurement.

Unlike the Modal load tests, this runs locally with:
- No network latency (localhost)
- Direct GPU monitoring (nvidia-smi + torch.cuda)
- Higher concurrency ceiling (single-machine, no rate limits)
- Real streaming TTFB measurement (byte-level timing)

Usage:
    # Full suite (ramps 1 → 5 → 10 → 20 → 50 → 100 concurrent)
    python scripts/stress_test_local.py

    # Quick smoke test (1 → 5 → 10 only)
    python scripts/stress_test_local.py --quick

    # Custom ramp
    python scripts/stress_test_local.py --levels 1,5,10,20

    # Streaming only
    python scripts/stress_test_local.py --stream-only

    # Non-streaming only
    python scripts/stress_test_local.py --no-stream

    # Custom endpoint
    python scripts/stress_test_local.py --url http://localhost:8080
"""

import argparse
import asyncio
import json
import os
import statistics
import subprocess
import sys
import time
import threading
from dataclasses import dataclass, field
from typing import List, Optional, Dict, Any, Tuple

import httpx

# === Configuration ===

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

# Test texts at different lengths to stress different parts of the pipeline
TEST_TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This sentence tests basic text normalization and generation quality "
        "with moderate length input that should complete in reasonable time."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast long shadows "
        "over crowded streets, there lived an old bookkeeper named Margaret. She had spent "
        "forty years cataloging the stories of others, yet never found time to write her own. "
        "One rainy Tuesday afternoon, she discovered a leather-bound journal tucked behind "
        "a shelf of dusty encyclopedias. Its pages were blank, waiting. She picked up her pen "
        "and began to write. The words flowed like water from a broken dam."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

# === GPU Monitoring ===

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    """Background thread that polls nvidia-smi for GPU stats."""

    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=5)

    def _poll_loop(self):
        while self._running:
            try:
                result = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True, text=True, timeout=5,
                )
                if result.returncode == 0:
                    parts = result.stdout.strip().split(", ")
                    if len(parts) >= 4:
                        self.snapshots.append(GPUSnapshot(
                            timestamp=time.time(),
                            memory_used_mb=float(parts[0]),
                            memory_total_mb=float(parts[1]),
                            gpu_utilization_pct=float(parts[2]),
                            temperature_c=float(parts[3]),
                        ))
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {"error": "no GPU data collected"}

        mem_used = [s.memory_used_mb for s in self.snapshots]
        gpu_util = [s.gpu_utilization_pct for s in self.snapshots]
        temps = [s.temperature_c for s in self.snapshots]

        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem_used),
                "max": max(mem_used),
                "avg": statistics.mean(mem_used),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(gpu_util),
                "max": max(gpu_util),
                "avg": statistics.mean(gpu_util),
            },
            "temperature_c": {
                "min": min(temps),
                "max": max(temps),
                "avg": statistics.mean(temps),
            },
        }

# === Request Result ===

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float  # Total wall-clock time
    ttfb_ms: float = 0.0  # Time to first byte (from HTTP response start)
    server_ttfb_ms: float = 0.0  # Server-reported TTFB (X-TTFB-ms header)
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    error: Optional[str] = None
    speaker: str = ""
    text_length: int = 0
    stream: bool = False

# === Load Level Result ===

@dataclass
class LevelResult:
    level_name: str
    concurrency: int
    total_requests: int
    mode: str  # "non-streaming" or "streaming"
    text_category: str  # "short", "medium", "long", "mixed"
    results: List[RequestResult] = field(default_factory=list)
    wall_time_seconds: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_seconds if self.wall_time_seconds > 0 else 0

    def _percentile(self, values: List[float], pct: float) -> float:
        if not values:
            return 0.0
        s = sorted(values)
        idx = int(len(s) * pct / 100)
        return s[min(idx, len(s) - 1)]

    @property
    def latency_stats(self) -> Dict[str, float]:
        lats = [r.latency_ms for r in self.successes]
        if not lats:
            return {}
        return {
            "min": min(lats),
            "avg": statistics.mean(lats),
            "p50": self._percentile(lats, 50),
            "p95": self._percentile(lats, 95),
            "p99": self._percentile(lats, 99),
            "max": max(lats),
        }

    @property
    def ttfb_stats(self) -> Dict[str, float]:
        ttfbs = [r.ttfb_ms for r in self.successes if r.ttfb_ms > 0]
        if not ttfbs:
            return {}
        return {
            "min": min(ttfbs),
            "avg": statistics.mean(ttfbs),
            "p50": self._percentile(ttfbs, 50),
            "p95": self._percentile(ttfbs, 95),
            "max": max(ttfbs),
        }

    @property
    def audio_duration_total(self) -> float:
        return sum(r.audio_seconds for r in self.successes)

    @property
    def effective_rtf(self) -> float:
        """Wall time / total audio produced. Lower = better."""
        total_audio = self.audio_duration_total
        return self.wall_time_seconds / total_audio if total_audio > 0 else 0

    def error_summary(self) -> Dict[str, int]:
        errors: Dict[str, int] = {}
        for r in self.failures:
            key = r.error or f"HTTP {r.status_code}"
            errors[key[:80]] = errors.get(key[:80], 0) + 1
        return errors

# === HTTP Client ===

async def make_request(
    client: httpx.AsyncClient,
    base_url: str,


sed -n '260,560p' /home/ubuntu/veenaModal/scripts/stress_test_local.py


Output:
Chunk ID: fa3a60
Wall time: 0.0521 seconds
Process exited with code 0
Original token count: 2666
Output:
    base_url: str,
    text: str,
    speaker: str,
    stream: bool = False,
    timeout: float = 120.0,
) -> RequestResult:
    """Make a single TTS request with timing."""
    url = f"{base_url}{GENERATE_PATH}"
    payload = {
        "text": text,
        "speaker": speaker,
        "stream": stream,
        "format": "wav",
    }

    start = time.time()
    ttfb_time = 0.0

    try:
        if stream:
            # Streaming: measure true TTFB (time to first byte of audio)
            async with client.stream(
                "POST", url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            ) as response:
                first_byte = True
                total_bytes = 0
                async for chunk in response.aiter_bytes():
                    if first_byte:
                        ttfb_time = (time.time() - start) * 1000
                        first_byte = False
                    total_bytes += len(chunk)

                latency = (time.time() - start) * 1000

                if response.status_code == 200:
                    # Estimate audio duration from bytes (16kHz, 16-bit, mono, minus WAV header)
                    pcm_bytes = max(0, total_bytes - 44)
                    audio_seconds = pcm_bytes / (16000 * 2)

                    return RequestResult(
                        success=True,
                        status_code=200,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        audio_bytes=total_bytes,
                        audio_seconds=audio_seconds,
                        speaker=speaker,
                        text_length=len(text),
                        stream=True,
                    )
                else:
                    return RequestResult(
                        success=False,
                        status_code=response.status_code,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        error=f"HTTP {response.status_code}",
                        speaker=speaker,
                        text_length=len(text),
                        stream=True,
                    )
        else:
            # Non-streaming: standard request
            response = await client.post(
                url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            )
            latency = (time.time() - start) * 1000

            # Server-reported TTFB
            server_ttfb = 0.0
            ttfb_header = response.headers.get("X-TTFB-ms")
            if ttfb_header:
                try:
                    server_ttfb = float(ttfb_header)
                except ValueError:
                    pass

            # Audio duration from header or estimate
            audio_seconds = 0.0
            audio_sec_header = response.headers.get("X-Audio-Seconds")
            if audio_sec_header:
                try:
                    audio_seconds = float(audio_sec_header)
                except ValueError:
                    pass
            elif response.status_code == 200:
                pcm_bytes = max(0, len(response.content) - 44)
                audio_seconds = pcm_bytes / (16000 * 2)

            if response.status_code == 200:
                return RequestResult(
                    success=True,
                    status_code=200,
                    latency_ms=latency,
                    ttfb_ms=latency,  # Non-streaming: TTFB = total latency
                    server_ttfb_ms=server_ttfb,
                    audio_bytes=len(response.content),
                    audio_seconds=audio_seconds,
                    speaker=speaker,
                    text_length=len(text),
                    stream=False,
                )
            else:
                error_text = response.text[:200] if response.text else f"HTTP {response.status_code}"
                return RequestResult(
                    success=False,
                    status_code=response.status_code,
                    latency_ms=latency,
                    error=error_text,
                    speaker=speaker,
                    text_length=len(text),
                    stream=False,
                )

    except httpx.TimeoutException:
        return RequestResult(
            success=False, status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error="TIMEOUT",
            speaker=speaker, text_length=len(text), stream=stream,
        )
    except Exception as e:
        return RequestResult(
            success=False, status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error=str(e)[:200],
            speaker=speaker, text_length=len(text), stream=stream,
        )

# === Load Runner ===

async def run_level(
    base_url: str,
    num_requests: int,
    concurrency: int,
    text_category: str = "short",
    stream: bool = False,
    gpu_monitor: Optional[GPUMonitor] = None,
) -> LevelResult:
    """Run a single load level and collect results."""
    mode = "streaming" if stream else "non-streaming"
    level_name = f"{concurrency}c-{text_category}-{mode}"

    level = LevelResult(
        level_name=level_name,
        concurrency=concurrency,
        total_requests=num_requests,
        mode=mode,
        text_category=text_category,
    )

    semaphore = asyncio.Semaphore(concurrency)

    # Determine text for each request
    if text_category == "mixed":
        texts = list(TEST_TEXTS.values())
    else:
        texts = [TEST_TEXTS.get(text_category, TEST_TEXTS["short"])]

    async def limited_request(i: int, client: httpx.AsyncClient) -> RequestResult:
        async with semaphore:
            text = texts[i % len(texts)]
            speaker = SPEAKERS[i % len(SPEAKERS)]
            return await make_request(client, base_url, text, speaker, stream=stream)

    # Start GPU monitoring for this level
    if gpu_monitor:
        gpu_monitor.snapshots.clear()

    start_time = time.time()

    # Use connection pooling for realistic load
    limits = httpx.Limits(
        max_connections=concurrency + 10,
        max_keepalive_connections=concurrency,
    )
    async with httpx.AsyncClient(limits=limits) as client:
        tasks = [limited_request(i, client) for i in range(num_requests)]
        results = await asyncio.gather(*tasks, return_exceptions=True)

    level.wall_time_seconds = time.time() - start_time

    for r in results:
        if isinstance(r, Exception):
            level.results.append(RequestResult(
                success=False, status_code=0, latency_ms=0,
                error=str(r)[:200], stream=stream,
            ))
        elif isinstance(r, RequestResult):
            level.results.append(r)

    if gpu_monitor:
        level.gpu_summary = gpu_monitor.summary()

    return level

# === Reporting ===

def print_level_report(level: LevelResult):
    """Print a compact report for one load level."""
    lat = level.latency_stats
    ttfb = level.ttfb_stats

    print(f"\n{'=' * 72}")
    print(f"  {level.level_name.upper()}")
    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}")
    print(f"{'=' * 72}")

    print(f"  Success:     {len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"  Wall time:   {level.wall_time_seconds:.2f}s")
    print(f"  Throughput:  {level.throughput_rps:.2f} req/s")
    print(f"  Audio total: {level.audio_duration_total:.1f}s")
    print(f"  Eff. RTF:    {level.effective_rtf:.3f}")

    if lat:
        print(f"\n  Latency (ms):")
        print(f"    min={lat['min']:.0f}  avg={lat['avg']:.0f}  "
              f"p50={lat['p50']:.0f}  p95={lat['p95']:.0f}  "
              f"p99={lat['p99']:.0f}  max={lat['max']:.0f}")

    if ttfb and level.mode == "streaming":
        print(f"  TTFB (ms):")
        print(f"    min={ttfb['min']:.0f}  avg={ttfb['avg']:.0f}  "
              f"p50={ttfb['p50']:.0f}  p95={ttfb['p95']:.0f}  max={ttfb['max']:.0f}")

    if level.gpu_summary and "memory_mb" in level.gpu_summary:
        gpu = level.gpu_summary
        mem = gpu["memory_mb"]
        util = gpu["gpu_util_pct"]
        print(f"\n  GPU:")
        print(f"    Memory: {mem['avg']:.0f}MB avg, {mem['max']:.0f}MB peak / {mem['total']:.0f}MB")
        print(f"    Util:   {util['avg']:.0f}% avg, {util['max']:.0f}% peak")
        print(f"    Temp:   {gpu['temperature_c']['avg']:.0f}C avg, {gpu['temperature_c']['max']:.0f}C peak")

    errors = level.error_summary()
    if errors:
        print(f"\n  Errors:")
        for err, count in sorted(errors.items(), key=lambda x: -x[1]):
            print(f"    [{count}x] {err}")

def print_summary_table(all_levels: List[LevelResult]):
    """Print a compact comparison table across all levels."""
    print(f"\n{'=' * 100}")
    print(f"  STRESS TEST SUMMARY")
    print(f"{'=' * 100}")

    header = f"{'Level':<28} {'OK%':>5} {'RPS':>7} {'p50':>7} {'p95':>7} {'p99':>7} {'RTF':>6} {'GPU%':>5} {'GPUmem':>8}"
    print(header)
    print("-" * 100)

    for level in all_levels:
        lat = level.latency_stats
        gpu_util_avg = level.gpu_summary.get("gpu_util_pct", {}).get("avg", 0)
        gpu_mem_max = level.gpu_summary.get("memory_mb", {}).get("max", 0)

        p50 = f"{lat.get('p50', 0):.0f}" if lat else "---"
        p95 = f"{lat.get('p95', 0):.0f}" if lat else "---"
        p99 = f"{lat.get('p99', 0):.0f}" if lat else "---"

        print(
            f"{level.level_name:<28} "
            f"{level.success_rate:>4.0%} "
            f"{level.throughput_rps:>7.2f} "
            f"{p50:>7} "
            f"{p95:>7} "
            f"{p99:>7} "
            f"{level.effective_rtf:>6.3f} "
            f"{gpu_util_avg:>4.0f}% "
            f"{gpu_mem_max:>6.0f}MB"
        )

    print("=" * 100)

# === Main ===

async def run_stress_test(args):
    """Run the full stress test suite."""
    base_url = args.url

    # Verify server is up
    print(f"Checking server at {base_url}...")
    try:
        async with httpx.AsyncClient() as client:
            resp = await client.get(f"{base_url}/v1/tts/health", timeout=10)
            health = resp.json()
            print(f"  Status: {health.get('status')}")
            print(f"  Model:  {health.get('model_version')}")
            print(f"  GPU:    {health.get('gpu_available')}")
            if health.get("status") != "healthy":
                print("WARNING: Server not fully healthy, results may be unreliable")
    except Exception as e:


sed -n '560,840p' /home/ubuntu/veenaModal/scripts/stress_test_local.py


Output:
Chunk ID: a8f048
Wall time: 0.0517 seconds
Process exited with code 0
Original token count: 1089
Output:
    except Exception as e:
        print(f"ERROR: Cannot reach server at {base_url}: {e}")
        sys.exit(1)

    # Parse concurrency levels
    levels = [int(x) for x in args.levels.split(",")]

    # Determine test modes
    test_modes = []
    if not args.stream_only:
        test_modes.append(False)  # non-streaming
    if not args.no_stream:
        test_modes.append(True)  # streaming

    # GPU monitor
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    all_results: List[LevelResult] = []

    # Warmup
    print(f"\nWarming up (2 requests)...")
    warmup = await run_level(base_url, num_requests=2, concurrency=1, text_category="short", stream=False)
    if warmup.success_rate < 1.0:
        print(f"  WARNING: Warmup had failures ({warmup.success_rate:.0%} success)")
    else:
        lat = warmup.latency_stats
        print(f"  OK ({lat.get('avg', 0):.0f}ms avg)")

    # Run each level
    for stream in test_modes:
        mode_label = "STREAMING" if stream else "NON-STREAMING"
        print(f"\n{'#' * 72}")
        print(f"  {mode_label} TESTS")
        print(f"{'#' * 72}")

        for conc in levels:
            # Scale requests: enough to get meaningful stats, cap at reasonable total
            num_requests = max(conc * 3, 10)  # At least 3x concurrency or 10
            num_requests = min(num_requests, 200)  # Cap at 200

            text_cat = args.text if args.text != "mixed" else "mixed"

            print(f"\n>>> {conc} concurrent, {num_requests} total requests, text={text_cat}, stream={stream}")

            level_result = await run_level(
                base_url=base_url,
                num_requests=num_requests,
                concurrency=conc,
                text_category=text_cat,
                stream=stream,
                gpu_monitor=gpu_monitor,
            )

            print_level_report(level_result)
            all_results.append(level_result)

            # Brief cooldown between levels
            if conc < levels[-1]:
                print(f"\n  Cooling down 3s...")
                await asyncio.sleep(3)

    gpu_monitor.stop()

    # Summary
    print_summary_table(all_results)

    # Save results to JSON
    output_path = args.output
    if output_path:
        save_results(all_results, output_path)
        print(f"\nResults saved to {output_path}")

def save_results(all_results: List[LevelResult], path: str):
    """Save results as JSON for later analysis."""
    data = []
    for level in all_results:
        data.append({
            "level": level.level_name,
            "concurrency": level.concurrency,
            "total_requests": level.total_requests,
            "mode": level.mode,
            "text_category": level.text_category,
            "success_rate": level.success_rate,
            "throughput_rps": level.throughput_rps,
            "wall_time_s": level.wall_time_seconds,
            "latency_ms": level.latency_stats,
            "ttfb_ms": level.ttfb_stats,
            "effective_rtf": level.effective_rtf,
            "audio_total_seconds": level.audio_duration_total,
            "errors": level.error_summary(),
            "gpu": level.gpu_summary,
        })

    with open(path, "w") as f:
        json.dump(data, f, indent=2, default=str)

def main():
    parser = argparse.ArgumentParser(
        description="Veena3 TTS Local Stress Test",
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument("--url", default=DEFAULT_URL, help=f"Server URL (default: {DEFAULT_URL})")
    parser.add_argument("--levels", default="1,5,10,20,50", help="Comma-separated concurrency levels (default: 1,5,10,20,50)")
    parser.add_argument("--quick", action="store_true", help="Quick mode: 1,5,10 only")
    parser.add_argument("--text", default="short", choices=["short", "medium", "long", "mixed"], help="Text category (default: short)")
    parser.add_argument("--stream-only", action="store_true", help="Only test streaming")
    parser.add_argument("--no-stream", action="store_true", help="Only test non-streaming")
    parser.add_argument("--output", default="stress_test_results.json", help="Output JSON path")

    args = parser.parse_args()

    if args.quick:
        args.levels = "1,5,10"

    asyncio.run(run_stress_test(args))

if __name__ == "__main__":
    main()


sed -n '1,260p' /home/ubuntu/veenaModal/scripts/measure_ttfb_detailed_v2.py


Output:
Chunk ID: dd6f70
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 719
Output:
#!/usr/bin/env python3
"""
Measure TTFB with detailed per-chunk timing
"""

import requests
import time

API_URL = "http://localhost:8000/v1/tts/generate"

def measure_streaming_with_all_chunks(text="Hello world, this is a test.", speaker="lipakshi"):
    """
    Measure TTFB showing ALL chunks received
    """
    print("\n" + "="*80)
    print("📊 DETAILED PER-CHUNK MEASUREMENT")
    print("="*80)

    payload = {
        "text": text,
        "speaker": speaker,
        "stream": True,
        "format": "wav"
    }

    headers = {"Content-Type": "application/json"}

    t0 = time.time()
    print(f"\n⏰ T+{0:.3f}s - Request initiated")

    response = requests.post(API_URL, json=payload, headers=headers, stream=True, timeout=30)

    t1 = time.time() - t0
    print(f"⏰ T+{t1:.3f}s - Headers received (HTTP {response.status_code})")

    if response.status_code != 200:
        print(f"❌ Error: {response.status_code}")
        return

    chunk_times = []
    chunk_sizes = []
    total_bytes = 0

    print(f"\n📦 Receiving chunks:")
    for i, chunk in enumerate(response.iter_content(chunk_size=None)):
        if chunk:
            t_chunk = time.time() - t0
            chunk_times.append(t_chunk)
            chunk_sizes.append(len(chunk))
            total_bytes += len(chunk)

            # Show first 10 chunks and then every 10th
            if i < 10 or i % 10 == 0:
                content_type = ""
                if i == 0 and chunk.startswith(b'RIFF'):
                    content_type = " [WAV header]"
                elif i == 0:
                    content_type = " [Audio data]"

                print(f"   #{i+1:3d} @ T+{t_chunk:.3f}s: {len(chunk):>6,} bytes{content_type}")

    t_end = time.time() - t0

    print(f"\n📊 SUMMARY:")
    print(f"   ├─ Total chunks:          {len(chunk_times)}")
    print(f"   ├─ Total bytes:           {total_bytes:,}")
    print(f"   ├─ First chunk (TTFB):    T+{chunk_times[0]:.3f}s = {chunk_times[0]*1000:.0f}ms")
    if len(chunk_times) > 1:
        print(f"   ├─ Second chunk:          T+{chunk_times[1]:.3f}s = {chunk_times[1]*1000:.0f}ms")
        print(f"   ├─ Delta (chunk 1→2):     {(chunk_times[1] - chunk_times[0])*1000:.0f}ms")
    print(f"   └─ Total time:            T+{t_end:.3f}s = {t_end*1000:.0f}ms")

    print(f"\n💡 INSIGHT:")
    if chunk_sizes[0] == 44:
        print(f"   First chunk is WAV header (44 bytes) - true audio starts at chunk #2")
        if len(chunk_times) > 1:
            print(f"   TRUE HEARABLE AUDIO TTFB: {chunk_times[1]*1000:.0f}ms")
    else:
        print(f"   First chunk contains audio data")
        print(f"   TRUE HEARABLE AUDIO TTFB: {chunk_times[0]*1000:.0f}ms")

if __name__ == "__main__":
    measure_streaming_with_all_chunks()


sed -n '1,320p' /home/ubuntu/veenaModal/veena3modal/app.py


Output:
Chunk ID: d5257a
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 2574
Output:
"""
Modal entrypoint for Veena3 TTS service.

Deploys a GPU-backed ASGI app with:
- True streaming TTS
- Autoscaling (0 to N containers)
- Per-container concurrency (vLLM batching)
- Memory snapshots for fast cold starts

Usage:
    # Deploy to Modal
    modal deploy veena3modal/app.py

    # Serve locally (for testing)
    modal serve veena3modal/app.py

    # Run a single function
    modal run veena3modal/app.py::tts_api
"""

from __future__ import annotations

import os
import modal

# Get the repo root for adding local sources
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# === Image Definition ===
# Full dependencies for TTS inference

image = (
    modal.Image.debian_slim(python_version="3.10")
    .apt_install(
        "ffmpeg",  # Audio encoding (opus, mp3, flac, mulaw)
        "git",     # For pip git installs
    )
    .pip_install(
        # Core ML/AI
        "torch>=2.0.0",
        "torchaudio>=2.0.0",
        "transformers>=4.35.0",
        "vllm>=0.4.0",

        # FastAPI
        "fastapi[standard]>=0.100.0",
        "uvicorn[standard]>=0.23.0",

        # Audio processing
        "numpy>=1.24.0",
        "scipy>=1.10.0",
        "librosa>=0.10.0",
        "soundfile>=0.12.0",

        # Utils
        "pydantic>=2.0.0",
        "prometheus_client>=0.17.0",
        "supabase>=2.0.0",

        # vLLM dependencies
        "einops>=0.6.0",
        "einx>=0.2.0",

        # SparkTTS dependencies (from external/sparktts/requirements.txt)
        "omegaconf>=2.3.0",
        "safetensors>=0.5.0",
        "soxr>=0.5.0",

        # AP-BWE (super-resolution) dependencies
        "matplotlib>=3.8.0",
        "natsort>=8.0.0",
        "joblib>=1.0.0",
    )
    .run_commands(
        # Ensure CUDA is properly configured
        "pip install --no-cache-dir nvidia-ml-py3 || true"
    )
    # Environment settings
    .env({
        "AUTH_BYPASS_MODE": "true",  # Disable auth for testing
        # sparktts is at /root/external/sparktts, AP-BWE at /root/external/AP-BWE
        "PYTHONPATH": "/root/external/sparktts:/root/external/AP-BWE:/root",
        "_IMAGE_BUILD_VERSION": "2025-12-25-sr-chunking-support",  # Add SR support to generate_speech_chunked
    })
    # Add sparktts package - create proper structure by copying files
    # The original has symlinks that don't work with Modal mounts
    .run_commands(
        # Create sparktts package directory
        "mkdir -p /root/external/sparktts/sparktts/models",
        "mkdir -p /root/external/sparktts/sparktts/utils",
    )
    .add_local_dir(
        local_path=os.path.join(REPO_ROOT, "external", "sparktts", "sparktts"),
        remote_path="/root/external/sparktts/sparktts",
        copy=True,
    )
    .add_local_dir(
        local_path=os.path.join(REPO_ROOT, "external", "sparktts", "models"),
        remote_path="/root/external/sparktts/sparktts/models",  # Put models inside sparktts package
        copy=True,
    )
    .add_local_dir(
        local_path=os.path.join(REPO_ROOT, "external", "sparktts", "utils"),
        remote_path="/root/external/sparktts/sparktts/utils",  # Put utils inside sparktts package
        copy=True,
    )
    # Add AP-BWE for super-resolution (16kHz -> 48kHz)
    .add_local_dir(
        local_path=os.path.join(REPO_ROOT, "external", "AP-BWE"),
        remote_path="/root/external/AP-BWE",
        copy=True,
    )
    # Non-copy local mounts (MUST BE LAST - no build steps after this!)
    .add_local_python_source("veena3modal")
)

# === Volume for Model Weights ===
# Models should be uploaded to this volume before deployment

try:
    model_volume = modal.Volume.from_name("veena3-models", create_if_missing=True)
except Exception:
    model_volume = None  # Will fail at runtime if volume not available

# === Secrets (Optional) ===
# Secrets are optional - the app will run in bypass mode without them
# Create secrets in Modal dashboard: https://modal.com/secrets
# Or use modal.Secret.from_dotenv() for local testing

# === App Definition ===

app = modal.App(
    name="veena3-tts",
    image=image,
)

@app.cls(
    # === GPU Selection ===
    # L40S: Best cost/performance ratio for TTS inference
    # - 48GB VRAM, $1.50/hr (vs A100-80GB at $4.50/hr)
    # - Handles ~12 req/s with <2s p95 latency per container
    # - Memory-bound workload, so faster GPUs have diminishing returns
    gpu="L40S",

    # === CPU Allocation (OPTIMIZATION Dec 2025) ===
    # Explicitly reserve 4 vCPUs for the Python driver process
    # Prevents CPU throttling during vLLM scheduler + streaming loop
    # Modal default is only 0.125 cores which can bottleneck streaming
    cpu=4.0,

    volumes={"/models": model_volume} if model_volume else {},
    secrets=[modal.Secret.from_name("veena3-secrets")],  # Supabase credentials

    # === Autoscaling Configuration ===
    # See .cursor/scaling.md for detailed analysis
    #
    # LOAD TEST RESULTS (Dec 2025):
    # - Cold boot: ~49s (memory snapshot captures imports, not GPU model)
    # - MAX SUSTAINABLE: 280 req/s @ 250 concurrent users (P95=1417ms, 100% success)
    # - Breaking point: 300+ users (429 errors start)
    # - Per container: ~14 req/s (280/20 containers)
    #
    min_containers=1,        # Keep 1 warm 24/7 for instant response
    max_containers=50,       # ~280 req/s capacity at P95 < 3s
    buffer_containers=1,     # REQUIRED: 49s cold boot too slow for burst
    scaledown_window=300,    # 5 min idle before scaledown (user changed from 120)

    # === Timeouts ===
    timeout=120,             # 2 min max per request (reduced from 10 min)
    startup_timeout=600,     # 10 min for model loading (reduced from 20 min)

    # === Memory Optimization ===
    # Memory snapshot for faster cold boots
    # Note: GPU snapshot is experimental (alpha) - disabled for stability
    enable_memory_snapshot=True,
)
# === Per-Container Concurrency ===
# vLLM continuous batching handles concurrent requests efficiently
# Increased from 12 to 20 to maximize GPU utilization while keeping P95 < 3s
@modal.concurrent(max_inputs=20, target_inputs=15)
class TTSService:
    """
    TTS service class with model lifecycle management.

    Uses @modal.enter for model loading (once per container).
    Memory snapshot captures imports and non-GPU state for faster restarts.
    """

    @modal.enter()
    def load_model(self):
        """
        Load TTS model when container starts.

        Called once per container lifecycle.
        Memory snapshot captures import state but not GPU memory.
        """
        import os
        import logging

        logging.basicConfig(level=logging.INFO)
        logger = logging.getLogger(__name__)

        logger.info("🚀 Starting TTS model loading...")

        # Set model paths from Volume mount or env vars
        model_path = os.environ.get(
            "MODEL_PATH",
            "/models/spark_tts_4speaker"
        )
        sr_path = os.environ.get(
            "AP_BWE_CHECKPOINT_DIR",
            "/models/ap_bwe/16kto48k"
        )

        # Check if model exists
        if not os.path.exists(model_path):
            logger.warning(f"Model path not found: {model_path}")
            logger.warning("TTS will return 503 until model is available")
            return

        try:
            from veena3modal.services.tts_runtime import initialize_runtime

            # Initialize with GPU memory optimization
            self.runtime = initialize_runtime(
                model_path=model_path,
                sr_checkpoint_dir=sr_path if os.path.exists(sr_path) else None,
                device="cuda",
                gpu_memory_utilization=0.85,
                enable_sr=os.path.exists(sr_path),
            )

            logger.info(f"✅ TTS model loaded: {self.runtime.model_version}")

        except Exception as e:
            logger.error(f"❌ Model loading failed: {e}")
            import traceback
            traceback.print_exc()

    @modal.asgi_app()
    def serve(self):
        """
        Return FastAPI ASGI app for serving requests.
        """
        from veena3modal.api.fastapi_app import create_app
        return create_app()

# === Standalone Function (Alternative Deployment) ===
# Use this if you don't need @modal.enter lifecycle
# Note: Prefer TTSService class for production (better model lifecycle management)

@app.function(
    gpu="L40S",  # Best cost/performance for TTS
    cpu=4.0,     # Explicit CPU allocation (OPTIMIZATION Dec 2025)
    volumes={"/models": model_volume} if model_volume else {},
    secrets=[modal.Secret.from_name("veena3-secrets")],
    min_containers=0,        # Scale to zero (no min for standalone)
    max_containers=5,        # Lower cap for standalone
    buffer_containers=1,
    scaledown_window=180,    # 3 min
    timeout=120,             # 2 min max
    startup_timeout=600,     # 10 min for model load
)
@modal.asgi_app()
def tts_api():
    """
    Standalone ASGI app function.

    Note: Model will be loaded on first request (higher TTFB).
    For production, use TTSService class with @modal.enter.
    """
    from veena3modal.api.fastapi_app import create_app
    return create_app()

# === Health Check Function ===

@app.function(
    # No GPU needed for health check
)
def health_check():
    """
    Simple health check that can be called from Modal dashboard.
    """
    return {
        "status": "ok",
        "service": "veena3-tts",
        "version": "0.1.0",
    }

# === Debug Function ===

@app.function(
    gpu="L40S",
    volumes={"/models": model_volume} if model_volume else {},
    timeout=300,
)
def debug_imports():
    """
    Debug function to test import paths and file structure.
    Run with: modal run veena3modal/app.py::debug_imports
    """
    import os
    import sys

    results = {
        "pythonpath": os.environ.get("PYTHONPATH", "NOT_SET"),
        "sys_path": sys.path[:10],  # First 10 entries
        "files": {},
        "import_tests": {},
    }

    # Check key directories
    dirs_to_check = [
        "/root",
        "/root/external",
        "/root/external/sparktts",


sed -n '1,260p' /home/ubuntu/veenaModal/veena3modal/shared/logging.py


Output:
Chunk ID: 69d2bf
Wall time: 0.0505 seconds
Process exited with code 0
Original token count: 1755
Output:
"""
Structured logging for Veena3 TTS service.

Features:
- JSON-formatted logs for structured queries
- Request ID tracking via context
- Lifecycle event helpers (request_received, first_audio_emitted, etc.)
- NO PII in logs (text content stored in Supabase, not logs)

Usage:
    from veena3modal.shared.logging import get_logger, log_event, set_request_context

    logger = get_logger(__name__)
    set_request_context(request_id="req-123")
    log_event(logger, "request_received", text_length=100, speaker="lipakshi")
"""

import logging
import json
import sys
import traceback
from datetime import datetime, timezone
from typing import Any, Dict, Optional
from contextvars import ContextVar

# Context variable for request-scoped data
_request_context: ContextVar[Dict[str, Any]] = ContextVar('request_context', default={})

def set_request_context(request_id: Optional[str] = None, **kwargs) -> None:
    """
    Set request context for current async task.

    Args:
        request_id: Unique request identifier
        **kwargs: Additional context fields (user_id, etc.)
    """
    ctx = {"request_id": request_id, **kwargs}
    _request_context.set(ctx)

def get_request_context() -> Dict[str, Any]:
    """Get current request context."""
    return _request_context.get()

def clear_request_context() -> None:
    """Clear request context after request completes."""
    _request_context.set({})

class JSONFormatter(logging.Formatter):
    """
    JSON log formatter for structured logging.

    Output format:
    {
        "timestamp": "2024-01-01T12:00:00.000Z",
        "level": "INFO",
        "logger": "veena3modal.api",
        "message": "Request received",
        "request_id": "req-123",
        ...extra fields...
    }
    """

    # Fields to exclude from extra (already handled separately)
    RESERVED_ATTRS = {
        'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
        'funcName', 'levelname', 'levelno', 'lineno', 'module', 'msecs',
        'message', 'msg', 'name', 'pathname', 'process', 'processName',
        'relativeCreated', 'stack_info', 'thread', 'threadName',
        'taskName',  # Python 3.12+
    }

    def format(self, record: logging.LogRecord) -> str:
        """Format log record as JSON."""
        # Build base log entry
        log_entry = {
            "timestamp": datetime.now(timezone.utc).isoformat(),
            "level": record.levelname,
            "logger": record.name,
            "message": record.getMessage(),
        }

        # Add request context if available
        ctx = get_request_context()
        if ctx.get("request_id"):
            log_entry["request_id"] = ctx["request_id"]

        # Add extra fields from record
        for key, value in record.__dict__.items():
            if key not in self.RESERVED_ATTRS and not key.startswith('_'):
                try:
                    # Ensure value is JSON serializable
                    json.dumps(value)
                    log_entry[key] = value
                except (TypeError, ValueError):
                    log_entry[key] = str(value)

        # Add exception info if present
        if record.exc_info:
            log_entry["exception"] = ''.join(traceback.format_exception(*record.exc_info))

        return json.dumps(log_entry, default=str)

def get_logger(name: str) -> logging.Logger:
    """
    Get a logger configured for structured JSON output.

    Args:
        name: Logger name (typically __name__)

    Returns:
        Configured logger instance
    """
    logger = logging.getLogger(name)

    # Only configure if no handlers exist (avoid duplicate setup)
    if not logger.handlers:
        handler = logging.StreamHandler(sys.stdout)
        handler.setFormatter(JSONFormatter())
        logger.addHandler(handler)
        logger.setLevel(logging.INFO)

    return logger

def log_event(
    logger: logging.Logger,
    event_type: str,
    level: int = logging.INFO,
    request_id: Optional[str] = None,
    **kwargs
) -> None:
    """
    Log a structured event.

    Args:
        logger: Logger instance
        event_type: Event name (e.g., "request_received", "first_audio_emitted")
        level: Log level (default: INFO)
        request_id: Override request_id from context
        **kwargs: Event-specific fields
    """
    extra = {"event": event_type, **kwargs}

    if request_id:
        extra["request_id"] = request_id

    logger.log(level, event_type, extra=extra)

def create_lifecycle_event(
    event_type: str,
    request_id: str,
    **kwargs
) -> Dict[str, Any]:
    """
    Create a lifecycle event dictionary.

    Lifecycle events:
    - request_received: TTS request started
    - auth_validated: API key validated
    - generation_started: Model inference began
    - first_audio_emitted: First audio chunk sent (TTFB)
    - request_completed: Request finished successfully
    - request_failed: Request failed with error

    Args:
        event_type: Event name
        request_id: Request identifier
        **kwargs: Event-specific fields

    Returns:
        Event dictionary ready for logging
    """
    return {
        "event": event_type,
        "request_id": request_id,
        "timestamp": datetime.now(timezone.utc).isoformat(),
        **kwargs
    }

# Pre-configured logger for the TTS service
tts_logger = get_logger("veena3modal.tts")

def log_request_received(
    request_id: str,
    text_length: int,
    speaker: str,
    stream: bool,
    format: str,
    **kwargs
) -> None:
    """Log request_received lifecycle event."""
    log_event(
        tts_logger,
        "request_received",
        request_id=request_id,
        text_length=text_length,
        speaker=speaker,
        stream=stream,
        format=format,
        **kwargs
    )

def log_first_audio_emitted(
    request_id: str,
    ttfb_ms: int,
    chunk_size_bytes: int,
    **kwargs
) -> None:
    """Log first_audio_emitted lifecycle event (TTFB marker)."""
    log_event(
        tts_logger,
        "first_audio_emitted",
        request_id=request_id,
        ttfb_ms=ttfb_ms,
        chunk_size_bytes=chunk_size_bytes,
        **kwargs
    )

def log_request_completed(
    request_id: str,
    status_code: int,
    total_duration_ms: int,
    audio_duration_seconds: float,
    rtf: float,
    chunks_sent: int = 1,
    **kwargs
) -> None:
    """Log request_completed lifecycle event."""
    log_event(
        tts_logger,
        "request_completed",
        request_id=request_id,
        status_code=status_code,
        total_duration_ms=total_duration_ms,
        audio_duration_seconds=audio_duration_seconds,
        rtf=rtf,
        chunks_sent=chunks_sent,
        **kwargs
    )

def log_request_failed(
    request_id: str,
    status_code: int,
    error_code: str,
    error_message: str,
    **kwargs
) -> None:
    """Log request_failed lifecycle event."""
    log_event(


rg -n "def generate_with_chunking|chunked_metrics|return_timing|llm_token|batch" /home/ubuntu/veenaModal/veena3modal/processing/long_text_processor.py