Output:
Chunk ID: 580b2c
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 2271
Output:
#!/usr/bin/env python3
"""Detailed local stress benchmark for optimized TTS path.

This script is focused on collecting per-request GPU/LLM timing for local tuning:
- Request wall-clock latency and server reported TTFB
- GPU timing breakdown (LLM batch wall / GPU wall, parse, decode)
- Time per token and per batch
- Concurrency visibility from in-flight counters
- End-to-end throughput and RTF with GPU utilization snapshots

Usage:
    python scripts/stress_test_optimized_local.py
    python scripts/stress_test_optimized_local.py --levels 1,5,10,20 --concurrency-scaling both
    python scripts/stress_test_optimized_local.py --text-category long --output benchmarks.json
"""

import argparse
import asyncio
import json
import statistics
import subprocess
import sys
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

import httpx

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

TEST_TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This is a baseline quality sentence used for benchmarking local TTS latency, "
        "TTFB, and tokenization behavior under load."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast shadows over crowded streets, "
        "there lived a quiet linguist who believed every sentence carried a rhythm. "
        "She recorded conversations at dawn and dusk, trying to capture how meaning shifted with breath. "
        "By evening she would review them, matching sound to emotion, and every night she learned a new way to speak."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    """Background polling for quick GPU health snapshots."""

    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=3)

    def _poll_loop(self):
        while self._running:
            try:
                result = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True,
                    text=True,
                    timeout=5,
                )
                if result.returncode == 0 and result.stdout:
                    parts = [p.strip() for p in result.stdout.strip().split(",")]
                    if len(parts) >= 4:
                        self.snapshots.append(
                            GPUSnapshot(
                                timestamp=time.time(),
                                memory_used_mb=float(parts[0]),
                                memory_total_mb=float(parts[1]),
                                gpu_utilization_pct=float(parts[2]),
                                temperature_c=float(parts[3]),
                            )
                        )
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {
                "samples": 0,
                "error": "no GPU samples captured",
            }

        mem_used = [s.memory_used_mb for s in self.snapshots]
        gpu_util = [s.gpu_utilization_pct for s in self.snapshots]
        temp = [s.temperature_c for s in self.snapshots]

        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem_used),
                "max": max(mem_used),
                "avg": statistics.mean(mem_used),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(gpu_util),
                "max": max(gpu_util),
                "avg": statistics.mean(gpu_util),
            },
            "temperature_c": {
                "min": min(temp),
                "max": max(temp),
                "avg": statistics.mean(temp),
            },
        }

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float
    ttfb_ms: float = 0.0
    server_ttfb_ms: float = 0.0
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    request_id: str = ""
    error: Optional[str] = None
    speaker: str = ""
    text_length: int = 0
    stream: bool = False
    timing: Dict[str, Any] = field(default_factory=dict)
    request_inflight: Optional[int] = None

@dataclass
class LevelResult:
    level_name: str
    concurrency: int
    total_requests: int
    mode: str
    text_category: str
    chunking: bool
    results: List[RequestResult] = field(default_factory=list)
    wall_time_seconds: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0.0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_seconds if self.wall_time_seconds > 0 else 0.0

    @property
    def audio_duration_total(self) -> float:
        return sum(r.audio_seconds for r in self.successes)

    @property
    def effective_rtf(self) -> float:
        total_audio = self.audio_duration_total
        return self.wall_time_seconds / total_audio if total_audio > 0 else 0.0

    def _percentile(self, values: List[float], pct: float) -> float:
        if not values:
            return 0.0
        sorted_vals = sorted(values)
        idx = int(len(sorted_vals) * pct / 100)
        return sorted_vals[min(idx, len(sorted_vals) - 1)]

    @property
    def latency_stats(self) -> Dict[str, float]:
        lat = [r.latency_ms for r in self.successes]
        if not lat:
            return {}
        return {
            "min": min(lat),
            "avg": statistics.mean(lat),
            "p50": self._percentile(lat, 50),
            "p95": self._percentile(lat, 95),
            "p99": self._percentile(lat, 99),
            "max": max(lat),
        }

    @property
    def ttfb_stats(self) -> Dict[str, float]:
        vals = [r.server_ttfb_ms for r in self.successes if r.server_ttfb_ms >= 0]
        if not vals:
            return {}
        return {
            "min": min(vals),
            "avg": statistics.mean(vals),
            "p50": self._percentile(vals, 50),
            "p95": self._percentile(vals, 95),
            "max": max(vals),
        }

    def timing_stats(self, key: str) -> Dict[str, float]:
        values: List[float] = []
        for r in self.successes:
            v = r.timing.get(key)
            if isinstance(v, (int, float)):
                values.append(float(v))
        if not values:
            return {}
        return {
            "min": min(values),
            "avg": statistics.mean(values),
            "p50": self._percentile(values, 50),
            "p95": self._percentile(values, 95),
            "p99": self._percentile(values, 99),
            "max": max(values),
        }

    def timing_rate(self, numerator_key: str, denominator_key: str) -> Tuple[float, int]:
        num = 0.0
        den = 0
        for r in self.successes:
            n = r.timing.get(numerator_key)
            d = r.timing.get(denominator_key)
            if isinstance(n, (int, float)) and isinstance(d, (int, float)) and d > 0:
                num += float(n)
                den += int(d)
        if den == 0:
            return 0.0, 0
        return num / den, den

    def observed_inflight_max(self) -> Optional[int]:
        values = [r.request_inflight for r in self.successes if isinstance(r.request_inflight, int)]
        if not values:
            return None
        return max(values)

    def error_summary(self) -> Dict[str, int]:
        errors: Dict[str, int] = {}
        for r in self.failures:
            key = r.error or f"HTTP {r.status_code}"
            errors[key[:100]] = errors.get(key[:100], 0) + 1
        return errors

def _coerce_number(raw: str) -> Optional[float]:
    if raw is None:
        return None
    v = str(raw).strip()
    if v == "":
        return None


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


I’ve confirmed the core benchmark + endpoint/header changes exist. I’ll now inspect the rest of the benchmark script and the remaining app sections to finish any compatibility gaps, then patch both in one pass.


sed -n '260,520p' scripts/stress_test_optimized_local.py


rg -n "generate_speech_chunked|generate_speech_streaming|generate_speech\(" veena3modal/api/fastapi_app.py


sed -n '1,260p' scripts/stress_test_local.py


Output:
Chunk ID: f90f2a
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 2467
Output:

    def observed_inflight_max(self) -> Optional[int]:
        values = [r.request_inflight for r in self.successes if isinstance(r.request_inflight, int)]
        if not values:
            return None
        return max(values)

    def error_summary(self) -> Dict[str, int]:
        errors: Dict[str, int] = {}
        for r in self.failures:
            key = r.error or f"HTTP {r.status_code}"
            errors[key[:100]] = errors.get(key[:100], 0) + 1
        return errors

def _coerce_number(raw: str) -> Optional[float]:
    if raw is None:
        return None
    v = str(raw).strip()
    if v == "":
        return None
    try:
        if "." in v:
            return float(v)
        return float(int(v))
    except ValueError:
        try:
            return float(v)
        except ValueError:
            return None

def _normalize_time_ms(raw: Optional[str]) -> float:
    value = _coerce_number(raw)
    if value is None:
        return 0.0
    return float(value)

def _parse_perf_headers(headers: Any) -> Dict[str, Any]:
    """Parse server timing headers into numeric metric fields."""
    metrics: Dict[str, Any] = {}

    # Explicit compact payload.
    detail_payload = headers.get("x-perf-details") if hasattr(headers, "get") else None
    if detail_payload:
        try:
            parsed = json.loads(detail_payload)
            if isinstance(parsed, dict):
                metrics.update(parsed)
        except Exception:
            pass

    header_map = {
        "x-llm-token-total": "llm_token_total",
        "x-llm-batch-count": "llm_batch_count",
        "x-llm-batch-wall-ms": "llm_batch_wall_ms_total",
        "x-llm-batch-gpu-ms": "llm_batch_gpu_ms_total",
        "x-llm-batch-wall-ms-min": "llm_batch_wall_ms_min",
        "x-llm-batch-wall-ms-max": "llm_batch_wall_ms_max",
        "x-llm-batch-wall-ms-p50": "llm_batch_wall_ms_p50",
        "x-llm-batch-gpu-ms-min": "llm_batch_gpu_ms_min",
        "x-llm-batch-gpu-ms-max": "llm_batch_gpu_ms_max",
        "x-llm-batch-gpu-ms-p50": "llm_batch_gpu_ms_p50",
        "x-llm-decode-wall-ms": "llm_decode_wall_ms_total",
        "x-llm-decode-gpu-ms": "llm_decode_gpu_ms_total",
        "x-llm-decode-wall-ms-min": "llm_decode_wall_ms_min",
        "x-llm-decode-wall-ms-max": "llm_decode_wall_ms_max",
        "x-llm-decode-wall-ms-p50": "llm_decode_wall_ms_p50",
        "x-llm-decode-gpu-ms-min": "llm_decode_gpu_ms_min",
        "x-llm-decode-gpu-ms-max": "llm_decode_gpu_ms_max",
        "x-llm-decode-gpu-ms-p50": "llm_decode_gpu_ms_p50",
        "x-llm-decode-calls": "decode_calls",
        "x-llm-decode-cpu-ms": "llm_decode_cpu_ms",
        "x-llm-parse-ms": "llm_parse_ms",
        "x-llm-parse-avg-ms": "llm_parse_ms_avg",
        "x-llm-tokens-per-batch-min": "tokens_per_batch_min",
        "x-llm-tokens-per-batch-max": "tokens_per_batch_max",
        "x-llm-tokens-per-batch-p50": "tokens_per_batch_p50",
        "x-generation-ms": "generation_ms",
        "x-request-inflight": "request_inflight",
        "x-llm-text-chunked": "text_chunked",
        "x-llm-chunks-processed": "chunks_processed",
        "x-llm-time-per-token-ms": "llm_time_per_token_ms",
        "x-llm-time-per-batch-wall-ms": "llm_time_per_batch_wall_ms",
        "x-llm-time-per-batch-gpu-ms": "llm_time_per_batch_gpu_ms",
        "x-llm-parse-avg-ms": "llm_parse_ms_avg",
    }

    if hasattr(headers, "items"):
        for raw_key, raw_val in headers.items():
            lk = str(raw_key).lower()
            target_key = header_map.get(lk)
            if target_key is None:
                continue
            v = _coerce_number(raw_val)
            if v is not None:
                if target_key in {
                    "text_chunked",
                }:
                    metrics[target_key] = bool(v)
                else:
                    metrics[target_key] = int(v) if float(v).is_integer() else float(v)

    if "llm_token_total" in metrics and "generation_ms" in metrics and metrics["llm_token_total"]:
        try:
            metrics["llm_time_per_token_ms"] = float(metrics["generation_ms"]) / float(metrics["llm_token_total"])
        except Exception:
            pass

    if "llm_batch_count" in metrics and "generation_ms" in metrics and metrics["llm_batch_count"]:
        try:
            metrics["llm_time_per_batch_wall_ms"] = float(metrics["generation_ms"]) / float(metrics["llm_batch_count"])
        except Exception:
            pass

    if "llm_batch_count" in metrics and "llm_batch_gpu_ms_total" in metrics and metrics["llm_batch_count"]:
        try:
            metrics["llm_time_per_batch_gpu_ms"] = float(metrics["llm_batch_gpu_ms_total"]) / float(metrics["llm_batch_count"])
        except Exception:
            pass

    return metrics

async def make_request(
    client: httpx.AsyncClient,
    base_url: str,
    text: str,
    speaker: str,
    chunking: bool,
    stream: bool = False,
    timeout: float = 120.0,
) -> RequestResult:
    url = f"{base_url}{GENERATE_PATH}"
    payload = {
        "text": text,
        "speaker": speaker,
        "stream": stream,
        "chunking": chunking,
        "format": "wav",
    }

    start = time.time()
    ttfb_time = 0.0

    try:
        if stream:
            async with client.stream(
                "POST",
                url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            ) as response:
                first = True
                total_bytes = 0
                async for chunk in response.aiter_bytes():
                    if first:
                        ttfb_time = (time.time() - start) * 1000
                        first = False
                    total_bytes += len(chunk)

                latency = (time.time() - start) * 1000
                timing = _parse_perf_headers(response.headers)

                if response.status_code == 200:
                    pcm_bytes = max(0, total_bytes - 44)
                    audio_seconds = pcm_bytes / (16000 * 2)
                    return RequestResult(
                        success=True,
                        status_code=200,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        audio_bytes=total_bytes,
                        audio_seconds=audio_seconds,
                        request_id=response.headers.get("x-request-id", ""),
                        speaker=speaker,
                        text_length=len(text),
                        stream=True,
                        timing=timing,
                        request_inflight=timing.get("request_inflight"),
                        server_ttfb_ms=_normalize_time_ms(response.headers.get("x-ttfb-ms")),
                    )

                return RequestResult(
                    success=False,
                    status_code=response.status_code,
                    latency_ms=latency,
                    ttfb_ms=ttfb_time,
                    error=f"HTTP {response.status_code}",
                    speaker=speaker,
                    text_length=len(text),
                    stream=True,
                    timing=timing,
                    request_id=response.headers.get("x-request-id", ""),
                    request_inflight=timing.get("request_inflight"),
                )
        else:
            response = await client.post(
                url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            )
            latency = (time.time() - start) * 1000
            timing = _parse_perf_headers(response.headers)

            server_ttfb = _normalize_time_ms(response.headers.get("x-ttfb-ms"))
            audio_seconds = 0.0
            audio_sec_header = response.headers.get("x-audio-seconds")
            if audio_sec_header:
                parsed = _coerce_number(audio_sec_header)
                if parsed is not None:
                    audio_seconds = parsed
            elif response.status_code == 200:
                pcm_bytes = max(0, len(response.content) - 44)
                audio_seconds = pcm_bytes / (16000 * 2)

            if response.status_code == 200:
                return RequestResult(
                    success=True,
                    status_code=200,
                    latency_ms=latency,
                    ttfb_ms=latency,
                    server_ttfb_ms=server_ttfb,
                    audio_bytes=len(response.content),
                    audio_seconds=audio_seconds,
                    request_id=response.headers.get("x-request-id", ""),
                    speaker=speaker,
                    text_length=len(text),
                    stream=False,
                    timing=timing,
                    request_inflight=timing.get("request_inflight"),
                )

            return RequestResult(
                success=False,
                status_code=response.status_code,
                latency_ms=latency,
                ttfb_ms=latency,
                server_ttfb_ms=server_ttfb,
                audio_bytes=len(response.content),
                error=(response.text[:200] if response.text else f"HTTP {response.status_code}"),
                speaker=speaker,
                text_length=len(text),
                stream=False,
                timing=timing,
                request_id=response.headers.get("x-request-id", ""),
                request_inflight=timing.get("request_inflight"),
            )

    except httpx.TimeoutException:
        return RequestResult(
            success=False,
            status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error="TIMEOUT",
            speaker=speaker,
            text_length=len(text),
            stream=stream,


Output:
Chunk ID: 28b6ae
Wall time: 0.0505 seconds
Process exited with code 0
Original token count: 65
Output:
446:                audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
460:                audio_bytes, metrics = await tts_runtime.generate_speech(
682:                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(


Output:
Chunk ID: e2a220
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2059
Output:
#!/usr/bin/env python3
"""
Local Stress Test for Veena3 TTS Server.

Progressive load ramp with GPU monitoring, latency percentiles,
error categorization, and throughput measurement.

Unlike the Modal load tests, this runs locally with:
- No network latency (localhost)
- Direct GPU monitoring (nvidia-smi + torch.cuda)
- Higher concurrency ceiling (single-machine, no rate limits)
- Real streaming TTFB measurement (byte-level timing)

Usage:
    # Full suite (ramps 1 → 5 → 10 → 20 → 50 → 100 concurrent)
    python scripts/stress_test_local.py

    # Quick smoke test (1 → 5 → 10 only)
    python scripts/stress_test_local.py --quick

    # Custom ramp
    python scripts/stress_test_local.py --levels 1,5,10,20

    # Streaming only
    python scripts/stress_test_local.py --stream-only

    # Non-streaming only
    python scripts/stress_test_local.py --no-stream

    # Custom endpoint
    python scripts/stress_test_local.py --url http://localhost:8080
"""

import argparse
import asyncio
import json
import os
import statistics
import subprocess
import sys
import time
import threading
from dataclasses import dataclass, field
from typing import List, Optional, Dict, Any, Tuple

import httpx

# === Configuration ===

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

# Test texts at different lengths to stress different parts of the pipeline
TEST_TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This sentence tests basic text normalization and generation quality "
        "with moderate length input that should complete in reasonable time."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast long shadows "
        "over crowded streets, there lived an old bookkeeper named Margaret. She had spent "
        "forty years cataloging the stories of others, yet never found time to write her own. "
        "One rainy Tuesday afternoon, she discovered a leather-bound journal tucked behind "
        "a shelf of dusty encyclopedias. Its pages were blank, waiting. She picked up her pen "
        "and began to write. The words flowed like water from a broken dam."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

# === GPU Monitoring ===

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    """Background thread that polls nvidia-smi for GPU stats."""

    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=5)

    def _poll_loop(self):
        while self._running:
            try:
                result = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True, text=True, timeout=5,
                )
                if result.returncode == 0:
                    parts = result.stdout.strip().split(", ")
                    if len(parts) >= 4:
                        self.snapshots.append(GPUSnapshot(
                            timestamp=time.time(),
                            memory_used_mb=float(parts[0]),
                            memory_total_mb=float(parts[1]),
                            gpu_utilization_pct=float(parts[2]),
                            temperature_c=float(parts[3]),
                        ))
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {"error": "no GPU data collected"}

        mem_used = [s.memory_used_mb for s in self.snapshots]
        gpu_util = [s.gpu_utilization_pct for s in self.snapshots]
        temps = [s.temperature_c for s in self.snapshots]

        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem_used),
                "max": max(mem_used),
                "avg": statistics.mean(mem_used),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(gpu_util),
                "max": max(gpu_util),
                "avg": statistics.mean(gpu_util),
            },
            "temperature_c": {
                "min": min(temps),
                "max": max(temps),
                "avg": statistics.mean(temps),
            },
        }

# === Request Result ===

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float  # Total wall-clock time
    ttfb_ms: float = 0.0  # Time to first byte (from HTTP response start)
    server_ttfb_ms: float = 0.0  # Server-reported TTFB (X-TTFB-ms header)
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    error: Optional[str] = None
    speaker: str = ""
    text_length: int = 0
    stream: bool = False

# === Load Level Result ===

@dataclass
class LevelResult:
    level_name: str
    concurrency: int
    total_requests: int
    mode: str  # "non-streaming" or "streaming"
    text_category: str  # "short", "medium", "long", "mixed"
    results: List[RequestResult] = field(default_factory=list)
    wall_time_seconds: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_seconds if self.wall_time_seconds > 0 else 0

    def _percentile(self, values: List[float], pct: float) -> float:
        if not values:
            return 0.0
        s = sorted(values)
        idx = int(len(s) * pct / 100)
        return s[min(idx, len(s) - 1)]

    @property
    def latency_stats(self) -> Dict[str, float]:
        lats = [r.latency_ms for r in self.successes]
        if not lats:
            return {}
        return {
            "min": min(lats),
            "avg": statistics.mean(lats),
            "p50": self._percentile(lats, 50),
            "p95": self._percentile(lats, 95),
            "p99": self._percentile(lats, 99),
            "max": max(lats),
        }

    @property
    def ttfb_stats(self) -> Dict[str, float]:
        ttfbs = [r.ttfb_ms for r in self.successes if r.ttfb_ms > 0]
        if not ttfbs:
            return {}
        return {
            "min": min(ttfbs),
            "avg": statistics.mean(ttfbs),
            "p50": self._percentile(ttfbs, 50),
            "p95": self._percentile(ttfbs, 95),
            "max": max(ttfbs),
        }

    @property
    def audio_duration_total(self) -> float:
        return sum(r.audio_seconds for r in self.successes)

    @property
    def effective_rtf(self) -> float:
        """Wall time / total audio produced. Lower = better."""
        total_audio = self.audio_duration_total
        return self.wall_time_seconds / total_audio if total_audio > 0 else 0

    def error_summary(self) -> Dict[str, int]:
        errors: Dict[str, int] = {}
        for r in self.failures:
            key = r.error or f"HTTP {r.status_code}"
            errors[key[:80]] = errors.get(key[:80], 0) + 1
        return errors

# === HTTP Client ===

async def make_request(
    client: httpx.AsyncClient,
    base_url: str,


Summary: auto


sed -n '360,780p' veena3modal/api/fastapi_app.py


Output:
Chunk ID: c2d14b
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 4475
Output:
                        content=create_error_response(
                            code=ErrorCode.RATE_LIMIT_EXCEEDED,
                            message=f"Rate limit exceeded. Retry after {int(reset_after) + 1} seconds.",
                            request_id=request_id,
                        ),
                        headers=headers,
                    )

        # Parse and validate request body
        try:
            body = await request.json()
        except Exception as e:
            raise HTTPException(status_code=400, detail=f"Invalid JSON: {e}")

        try:
            req = TTSGenerateRequest(**body)
        except Exception as e:
            error_msg = str(e)
            # Extract validation error details
            return JSONResponse(
                status_code=400,
                content={
                    "error": {
                        "code": "VALIDATION_ERROR",
                        "message": error_msg,
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

        # Check if runtime is initialized
        if not tts_runtime.is_initialized():
            return JSONResponse(
                status_code=503,
                content={
                    "error": {
                        "code": "MODEL_NOT_LOADED",
                        "message": "TTS model not initialized. Please wait for warmup.",
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

        # Get resolved speaker
        speaker = req.get_resolved_speaker()

        # Get normalized text (apply text normalization if enabled)
        normalizer_func = None
        if req.normalize:
            try:
                from veena3modal.processing.text_normalizer import normalize_text
                normalizer_func = lambda t: normalize_text(t, verbose=req.normalize_verbose)
            except ImportError:
                logger.warning("TextNormalizer not available, skipping text normalization")

        text = req.get_normalized_text(normalizer_func=normalizer_func)

        # Set request context for structured logging
        set_request_context(request_id=request_id)

        # Log request received (lifecycle event + metrics)
        log_request_received(
            request_id=request_id,
            text_length=len(text),
            speaker=speaker,
            stream=req.stream,
            format=req.format,
        )
        metrics_request_received(speaker=speaker, stream=req.stream, format=req.format)

        # Streaming mode (M4)
        if req.stream:
            return await _handle_streaming_request(req, request_id, text, speaker, start_time)

        # Format encoding will be applied after generation (non-streaming only)
        # For streaming, only WAV is supported
        target_format = req.format

        try:
            # Get output sample rate - use_enum_values=True means req.output is already a string
            output_sr = req.output if hasattr(req, 'output') else "16khz"

            # Generate speech (non-streaming)
                if req.chunking:
                audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
                    text=text,
                    speaker=speaker,
                    temperature=req.temperature,
                    top_k=req.top_k,
                    top_p=req.top_p,
                    max_tokens=req.max_tokens,
                    repetition_penalty=req.repetition_penalty,
                    seed=req.seed,
                    sample_rate=req.sample_rate,
                    output_sample_rate=output_sr,
                    request_id=request_id,
                )
            else:
                audio_bytes, metrics = await tts_runtime.generate_speech(
                    text=text,
                    speaker=speaker,
                    temperature=req.temperature,
                    top_k=req.top_k,
                    top_p=req.top_p,
                    max_tokens=req.max_tokens,
                    repetition_penalty=req.repetition_penalty,
                    seed=req.seed,
                    output_sample_rate=output_sr,
                    request_id=request_id,
                )

            if audio_bytes is None:
                return JSONResponse(
                    status_code=500,
                    content={
                        "error": {
                            "code": "GENERATION_FAILED",
                            "message": "Audio generation failed. Check logs for details.",
                            "request_id": request_id,
                        }
                    },
                    headers={"X-Request-ID": request_id},
                )

            # Calculate RTF
            total_time = time.time() - start_time
            audio_duration = metrics.get("audio_duration_seconds", 0)
            rtf = total_time / audio_duration if audio_duration > 0 else 0

            # Build response headers
            runtime = tts_runtime.get_runtime()
            model_version = runtime.model_version if runtime else get_model_version()

            # Use actual output sample rate from metrics
            output_sample_rate = metrics.get("output_sample_rate", req.sample_rate)

            headers = {
                "X-Request-ID": request_id,
                "X-Model-Version": model_version,
                "X-Format": "wav",
                "X-Sample-Rate": str(output_sample_rate),
                "X-Stream": "false",
                "X-Audio-Bytes": str(len(audio_bytes)),
                "X-Audio-Seconds": f"{audio_duration:.2f}",
                "X-TTFB-ms": str(metrics.get("ttfb_ms", 0)),
                "X-RTF": f"{rtf:.3f}",
                "X-Text-Chunked": str(metrics.get("text_chunked", False)).lower(),
                "X-SR-Applied": str(metrics.get("sr_applied", False)).lower(),
            }
            headers.update(_build_perf_headers(metrics))

            if req.seed is not None:
                headers["X-Seed"] = str(req.seed)

            # Fire-and-forget sentence storage (non-blocking, after generation)
            sentence_store = _get_sentence_store()
            sentence_store.store_fire_and_forget(
                request_id=request_id,
                text=text,
                speaker=speaker,
                stream=False,
                format=req.format,
                temperature=req.temperature,
                top_k=req.top_k,
                top_p=req.top_p,
                max_tokens=req.max_tokens,
                repetition_penalty=req.repetition_penalty,
                seed=req.seed,
                text_chunked=metrics.get("text_chunked", False),
                ttfb_ms=metrics.get("ttfb_ms"),
                audio_duration_seconds=audio_duration,
            )

            # Log successful completion (lifecycle event + metrics)
            total_duration_ms = int(total_time * 1000)
            log_request_completed(
                request_id=request_id,
                status_code=200,
                total_duration_ms=total_duration_ms,
                audio_duration_seconds=audio_duration,
                rtf=rtf,
                chunks_sent=1,
            )
            metrics_request_completed(
                status_code=200,
                duration_seconds=total_time,
                speaker=speaker,
                stream=False,
            )
            record_ttfb(ttfb_seconds=metrics.get("ttfb_ms", 0) / 1000, speaker=speaker, stream=False)
            record_rtf(rtf=rtf, speaker=speaker)
            record_audio_duration(duration_seconds=audio_duration, speaker=speaker)

            clear_request_context()

            # Apply format encoding if not WAV
            media_type = "audio/wav"
            if target_format != AudioFormat.WAV.value:
                try:
                    from veena3modal.audio.encoder import (
                        OpusEncoder, MP3Encoder, MuLawEncoder, FLACEncoder
                    )

                    # Get sample rate from output (after potential SR)
                    sample_rate = metrics.get("output_sample_rate", 16000)

                    # Strip WAV header to get PCM data
                    pcm_data = audio_bytes[44:] if len(audio_bytes) > 44 else audio_bytes

                    if target_format == AudioFormat.OPUS.value:
                        encoder = OpusEncoder(sample_rate=sample_rate)
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/opus"
                    elif target_format == AudioFormat.MP3.value:
                        encoder = MP3Encoder(sample_rate=sample_rate)
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/mpeg"
                    elif target_format == AudioFormat.MULAW.value:
                        encoder = MuLawEncoder()
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/x-wav"
                    elif target_format == AudioFormat.FLAC.value:
                        encoder = FLACEncoder(sample_rate=sample_rate)
                        audio_bytes = encoder.encode(pcm_data, sample_rate)
                        media_type = "audio/flac"

                    headers["X-Audio-Bytes"] = str(len(audio_bytes))
                    logger.info(f"Encoded audio to {target_format}: {len(audio_bytes)} bytes")
                except ImportError as e:
                    logger.warning(f"Audio encoder not available: {e}")
                    # Fall back to WAV
                except Exception as e:
                    logger.error(f"Format encoding failed: {e}")
                    # Fall back to WAV

            return Response(
                content=audio_bytes,
                media_type=media_type,
                headers=headers,
            )

        except Exception as e:
            logger.exception(f"TTS generation error: {e}")

            # Log failure (lifecycle event + metrics)
            log_request_failed(
                request_id=request_id,
                status_code=500,
                error_code="INTERNAL_ERROR",
                error_message=str(e),
            )
            metrics_request_failed(
                status_code=500,
                error_code="INTERNAL_ERROR",
                speaker=speaker,
            )
            clear_request_context()

            return JSONResponse(
                status_code=500,
                content={
                    "error": {
                        "code": "INTERNAL_ERROR",
                        "message": str(e),
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

    async def _handle_streaming_request(req, request_id: str, text: str, speaker: str, start_time: float):
        """
        Handle streaming TTS request.

        Returns a StreamingResponse that yields WAV header + PCM chunks.
        True streaming: first bytes sent ASAP before full audio is generated.
        """
        from fastapi.responses import StreamingResponse
        from veena3modal.api.schemas import AudioFormat
        from veena3modal.services import tts_runtime

        # Check if streaming pipeline is available
        runtime = tts_runtime.get_runtime()
        if runtime is None or runtime.streaming_pipeline is None:
            return JSONResponse(
                status_code=503,
                content={
                    "error": {
                        "code": "STREAMING_UNAVAILABLE",
                        "message": "Streaming pipeline not initialized.",
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

        # Non-WAV formats not implemented yet (M5)
        if req.format != AudioFormat.WAV.value:
            return JSONResponse(
                status_code=501,
                content={
                    "error": {
                        "code": "FORMAT_NOT_IMPLEMENTED",
                        "message": f"Streaming format '{req.format}' not yet implemented. Use format=wav.",
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

        # Set request context for structured logging
        set_request_context(request_id=request_id)

        # Create async generator for streaming response
        async def audio_stream_generator():
            """Yields audio chunks as they're generated."""
            final_metrics = {}
            first_chunk_sent = False
            chunks_count = 0
            try:
                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
                    text=text,
                    speaker=speaker,
                    temperature=req.temperature,
                    top_k=req.top_k,
                    top_p=req.top_p,
                    max_tokens=req.max_tokens,
                    repetition_penalty=req.repetition_penalty,
                    seed=req.seed,
                    enable_chunking=req.chunking,
                    request_id=request_id,
                ):
                    final_metrics = metrics
                    chunks_count += 1

                    # Fire-and-forget sentence storage after first chunk (don't block TTFB)
                    if not first_chunk_sent:
                        first_chunk_sent = True
                        ttfb_ms = metrics.get("ttfb_ms", 0)

                        # Log first audio emitted (TTFB marker)
                        log_first_audio_emitted(
                            request_id=request_id,
                            ttfb_ms=ttfb_ms,
                            chunk_size_bytes=len(audio_chunk),
                        )
                        record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)

                        sentence_store = _get_sentence_store()
                        sentence_store.store_fire_and_forget(
                            request_id=request_id,
                            text=text,
                            speaker=speaker,
                            stream=True,
                            format=req.format,
                            temperature=req.temperature,
                            top_k=req.top_k,
                            top_p=req.top_p,
                            max_tokens=req.max_tokens,
                            repetition_penalty=req.repetition_penalty,
                            seed=req.seed,
                            text_chunked=metrics.get("text_chunked", False),
                            ttfb_ms=ttfb_ms,
                            # audio_duration_seconds not known yet for streaming
                        )

                    yield audio_chunk

                # Stream completed - log metrics
                total_time = time.time() - start_time
                audio_duration = final_metrics.get("audio_duration_seconds", 0)
                rtf = total_time / audio_duration if audio_duration > 0 else 0

                log_request_completed(
                    request_id=request_id,
                    status_code=200,
                    total_duration_ms=int(total_time * 1000),
                    audio_duration_seconds=audio_duration,
                    rtf=rtf,
                    chunks_sent=chunks_count,
                )
                metrics_request_completed(
                    status_code=200,
                    duration_seconds=total_time,
                    speaker=speaker,
                    stream=True,
                )
                record_rtf(rtf=rtf, speaker=speaker)
                record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
                record_chunks_sent(chunks=chunks_count, speaker=speaker)
                clear_request_context()
            except Exception as e:
                logger.exception(f"Streaming error: {e}")
                # Log streaming failure
                log_request_failed(
                    request_id=request_id,
                    status_code=500,
                    error_code="STREAMING_ERROR",
                    error_message=str(e),
                )
                metrics_request_failed(
                    status_code=500,
                    error_code="STREAMING_ERROR",
                    speaker=speaker,
                )
                clear_request_context()
                # Can't return error once streaming started
                # Best we can do is stop yielding
                return

        # Build response headers
        # Note: Some headers can't be set dynamically during streaming
        # TTFB and RTF will be logged but not in headers for streaming
        model_version = runtime.model_version if runtime else get_model_version()

        headers = {
            "X-Request-ID": request_id,
            "X-Model-Version": model_version,
            "X-Format": "wav",


sed -n '780,980p' veena3modal/api/fastapi_app.py


Output:
Chunk ID: a64984
Wall time: 0.0516 seconds
Process exited with code 0
Original token count: 211
Output:
            "X-Format": "wav",
            "X-Sample-Rate": str(req.sample_rate),
            "X-Stream": "true",
            "X-Chunking-Enabled": str(req.chunking).lower(),
            # Transfer-Encoding: chunked is set automatically by StreamingResponse
        }

        if req.seed is not None:
            headers["X-Seed"] = str(req.seed)

        return StreamingResponse(
            audio_stream_generator(),
            media_type="audio/wav",
            headers=headers,
        )

    # === WebSocket Support (M5b) ===
    try:
        from veena3modal.api.websocket_handler import add_websocket_routes
        add_websocket_routes(app)
        logger.info("WebSocket support enabled at /v1/tts/ws")
    except ImportError as e:
        logger.warning(f"WebSocket support not available: {e}")

    return app


nl -ba veena3modal/api/fastapi_app.py | sed -n '430,500p'


Output:
Chunk ID: 4102ec
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 884
Output:
   430	        metrics_request_received(speaker=speaker, stream=req.stream, format=req.format)
   431
   432	        # Streaming mode (M4)
   433	        if req.stream:
   434	            return await _handle_streaming_request(req, request_id, text, speaker, start_time)
   435
   436	        # Format encoding will be applied after generation (non-streaming only)
   437	        # For streaming, only WAV is supported
   438	        target_format = req.format
   439
   440	        try:
   441	            # Get output sample rate - use_enum_values=True means req.output is already a string
   442	            output_sr = req.output if hasattr(req, 'output') else "16khz"
   443
   444	            # Generate speech (non-streaming)
   445	                if req.chunking:
   446	                audio_bytes, metrics = await tts_runtime.generate_speech_chunked(
   447	                    text=text,
   448	                    speaker=speaker,
   449	                    temperature=req.temperature,
   450	                    top_k=req.top_k,
   451	                    top_p=req.top_p,
   452	                    max_tokens=req.max_tokens,
   453	                    repetition_penalty=req.repetition_penalty,
   454	                    seed=req.seed,
   455	                    sample_rate=req.sample_rate,
   456	                    output_sample_rate=output_sr,
   457	                    request_id=request_id,
   458	                )
   459	            else:
   460	                audio_bytes, metrics = await tts_runtime.generate_speech(
   461	                    text=text,
   462	                    speaker=speaker,
   463	                    temperature=req.temperature,
   464	                    top_k=req.top_k,
   465	                    top_p=req.top_p,
   466	                    max_tokens=req.max_tokens,
   467	                    repetition_penalty=req.repetition_penalty,
   468	                    seed=req.seed,
   469	                    output_sample_rate=output_sr,
   470	                    request_id=request_id,
   471	                )
   472
   473	            if audio_bytes is None:
   474	                return JSONResponse(
   475	                    status_code=500,
   476	                    content={
   477	                        "error": {
   478	                            "code": "GENERATION_FAILED",
   479	                            "message": "Audio generation failed. Check logs for details.",
   480	                            "request_id": request_id,
   481	                        }
   482	                    },
   483	                    headers={"X-Request-ID": request_id},
   484	                )
   485
   486	            # Calculate RTF
   487	            total_time = time.time() - start_time
   488	            audio_duration = metrics.get("audio_duration_seconds", 0)
   489	            rtf = total_time / audio_duration if audio_duration > 0 else 0
   490
   491	            # Build response headers
   492	            runtime = tts_runtime.get_runtime()
   493	            model_version = runtime.model_version if runtime else get_model_version()
   494
   495	            # Use actual output sample rate from metrics
   496	            output_sample_rate = metrics.get("output_sample_rate", req.sample_rate)
   497
   498	            headers = {
   499	                "X-Request-ID": request_id,
   500	                "X-Model-Version": model_version,


nl -ba veena3modal/api/websocket_handler.py | sed -n '220,520p'


Output:
Chunk ID: d18cad
Wall time: 0.0506 seconds
Process exited with code 0
Original token count: 3414
Output:
   220	    """
   221	    from veena3modal.api.auth import get_api_validator, hash_api_key
   222	    from veena3modal.api.rate_limiter import get_rate_limiter
   223	    from veena3modal.api.error_handlers import FeatureFlags
   224	    from veena3modal.api.schemas import resolve_speaker_name
   225	    from veena3modal.services import tts_runtime
   226
   227	    request_id = str(uuid.uuid4())
   228	    set_request_context(request_id=request_id)
   229
   230	    # Accept the WebSocket connection
   231	    await websocket.accept()
   232	    logger.info(f"WebSocket connection accepted: {request_id}")
   233
   234	    # Track cancellation
   235	    cancelled = False
   236
   237	    try:
   238	        # === AUTH CHECK (if enabled) ===
   239	        if FeatureFlags.is_auth_enabled() and api_key:
   240	            validator = get_api_validator()
   241	            key_hash = hash_api_key(api_key)
   242	            auth_result = validator.validate(key_hash)
   243
   244	            if not auth_result.is_valid:
   245	                await websocket.send_text(create_error_message(
   246	                    code=auth_result.error_code or "AUTH_FAILED",
   247	                    message=auth_result.error_message or "Authentication failed",
   248	                    request_id=request_id,
   249	                ))
   250	                await websocket.close(code=1008)  # Policy violation
   251	                return
   252
   253	            # Rate limit check
   254	            if FeatureFlags.is_rate_limiting_enabled():
   255	                rate_limiter = get_rate_limiter()
   256	                allowed, remaining, reset_after = rate_limiter.check(key_hash)
   257	                if not allowed:
   258	                    await websocket.send_text(create_error_message(
   259	                        code="RATE_LIMIT_EXCEEDED",
   260	                        message=f"Rate limit exceeded. Retry after {int(reset_after) + 1} seconds.",
   261	                        request_id=request_id,
   262	                    ))
   263	                    await websocket.close(code=1008)
   264	                    return
   265
   266	        # === WAIT FOR REQUEST ===
   267	        try:
   268	            # Wait for client to send request (with timeout)
   269	            raw_message = await asyncio.wait_for(
   270	                websocket.receive_text(),
   271	                timeout=30.0
   272	            )
   273	            message = json.loads(raw_message)
   274	        except asyncio.TimeoutError:
   275	            await websocket.send_text(create_error_message(
   276	                code="TIMEOUT",
   277	                message="No request received within 30 seconds",
   278	                request_id=request_id,
   279	            ))
   280	            await websocket.close(code=1000)
   281	            return
   282	        except json.JSONDecodeError as e:
   283	            await websocket.send_text(create_error_message(
   284	                code="INVALID_JSON",
   285	                message=f"Invalid JSON: {e}",
   286	                request_id=request_id,
   287	            ))
   288	            await websocket.close(code=1003)  # Unsupported data
   289	            return
   290
   291	        # Handle control messages
   292	        event_type = message.get("event", "request")
   293	        if event_type == "ping":
   294	            await websocket.send_text(json.dumps({"event": "pong"}))
   295	            # Wait for actual request
   296	            raw_message = await websocket.receive_text()
   297	            message = json.loads(raw_message)
   298
   299	        # === PARSE REQUEST ===
   300	        req = WSRequest.from_dict(message)
   301	        validation_error = req.validate()
   302	        if validation_error:
   303	            await websocket.send_text(create_error_message(
   304	                code="VALIDATION_ERROR",
   305	                message=validation_error,
   306	                request_id=request_id,
   307	            ))
   308	            await websocket.close(code=1003)
   309	            return
   310
   311	        # Resolve speaker
   312	        speaker = resolve_speaker_name(req.speaker)
   313
   314	        # Log request
   315	        log_request_received(
   316	            request_id=request_id,
   317	            text_length=len(req.text),
   318	            speaker=speaker,
   319	            stream=True,
   320	            format="wav",
   321	        )
   322	        metrics_request_received(speaker=speaker, stream=True, format="wav")
   323
   324	        # === CHECK MODEL ===
   325	        if not tts_runtime.is_initialized():
   326	            await websocket.send_text(create_error_message(
   327	                code="MODEL_NOT_LOADED",
   328	                message="TTS model not initialized",
   329	                request_id=request_id,
   330	            ))
   331	            await websocket.close(code=1011)  # Internal error
   332	            return
   333
   334	        runtime = tts_runtime.get_runtime()
   335	        model_version = runtime.model_version if runtime else "unknown"
   336
   337	        # === SEND HEADER INFO ===
   338	        await websocket.send_text(create_header_message(
   339	            request_id=request_id,
   340	            sample_rate=req.sample_rate,
   341	            model_version=model_version,
   342	        ))
   343
   344	        # === STREAM AUDIO ===
   345	        start_time = time.time()
   346	        chunks_sent = 0
   347	        total_bytes = 0
   348	        first_chunk_time = None
   349	        final_metrics = {}
   350
   351	        # Start a task to listen for cancel messages
   352	        async def listen_for_cancel():
   353	            nonlocal cancelled
   354	            try:
   355	                while not cancelled:
   356	                    try:
   357	                        msg = await asyncio.wait_for(
   358	                            websocket.receive_text(),
   359	                            timeout=0.1
   360	                        )
   361	                        data = json.loads(msg)
   362	                        if data.get("event") == "cancel":
   363	                            cancelled = True
   364	                            logger.info(f"Client requested cancel: {request_id}")
   365	                            break
   366	                        elif data.get("event") == "ping":
   367	                            await websocket.send_text(json.dumps({"event": "pong"}))
   368	                    except asyncio.TimeoutError:
   369	                        continue
   370	                    except Exception:
   371	                        break
   372	            except Exception:
   373	                pass
   374
   375	        cancel_task = asyncio.create_task(listen_for_cancel())
   376
   377	        try:
   378	            # Apply text normalization if enabled
   379	            text = req.text
   380	            if req.normalize:
   381	                try:
   382	                    from veena3modal.processing.text_normalizer import normalize_text
   383	                    text = normalize_text(text)
   384	                except ImportError:
   385	                    pass  # Normalizer not available
   386
   387	            # Stream audio chunks
   388	            async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
   389	                text=text,
   390	                speaker=speaker,
   391	                temperature=req.temperature,
   392	                top_k=req.top_k,
   393	                top_p=req.top_p,
   394	                max_tokens=req.max_tokens,
   395	                repetition_penalty=req.repetition_penalty,
   396	                seed=req.seed,
   397	                enable_chunking=req.chunking,
   398	                request_id=request_id,
   399	            ):
   400	                if cancelled:
   401	                    logger.info(f"Generation cancelled: {request_id}")
   402	                    break
   403
   404	                final_metrics = metrics
   405	                chunks_sent += 1
   406	                total_bytes += len(audio_chunk)
   407
   408	                # Record TTFB on first chunk
   409	                if first_chunk_time is None:
   410	                    first_chunk_time = time.time()
   411	                    ttfb_ms = int((first_chunk_time - start_time) * 1000)
   412	                    log_first_audio_emitted(
   413	                        request_id=request_id,
   414	                        ttfb_ms=ttfb_ms,
   415	                        chunk_size_bytes=len(audio_chunk),
   416	                    )
   417	                    record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)
   418
   419	                # Send audio as binary frame
   420	                await websocket.send_bytes(audio_chunk)
   421
   422	                # Send progress every 10 chunks
   423	                if chunks_sent % 10 == 0:
   424	                    await websocket.send_text(create_progress_message(
   425	                        chunks_sent=chunks_sent,
   426	                        bytes_sent=total_bytes,
   427	                        elapsed_ms=int((time.time() - start_time) * 1000),
   428	                    ))
   429
   430	        finally:
   431	            cancel_task.cancel()
   432	            try:
   433	                await cancel_task
   434	            except asyncio.CancelledError:
   435	                pass
   436
   437	        # === SEND COMPLETION ===
   438	        total_time = time.time() - start_time
   439	        audio_duration = final_metrics.get("audio_duration_seconds", 0)
   440	        rtf = total_time / audio_duration if audio_duration > 0 else 0
   441
   442	        completion_metrics = {
   443	            "request_id": request_id,
   444	            "chunks_sent": chunks_sent,
   445	            "total_bytes": total_bytes,
   446	            "audio_duration_seconds": round(audio_duration, 2),
   447	            "total_time_seconds": round(total_time, 2),
   448	            "ttfb_ms": int((first_chunk_time - start_time) * 1000) if first_chunk_time else 0,
   449	            "rtf": round(rtf, 3),
   450	            "cancelled": cancelled,
   451	        }
   452
   453	        if first_chunk_time is None:
   454	            completion_metrics["ttfb_ms"] = 0
   455
   456	        # Include request-scoped runtime timing for post-run profiling.
   457	        for key in sorted(_METRIC_FORWARD_KEYS):
   458	            if key in final_metrics:
   459	                completion_metrics[key] = final_metrics[key]
   460
   461	        if completion_metrics.get("audio_duration_seconds") in (0, 0.0) and final_metrics.get("audio_duration_seconds"):
   462	            completion_metrics["audio_duration_seconds"] = final_metrics.get("audio_duration_seconds")
   463
   464	        token_total = final_metrics.get("llm_token_total") or final_metrics.get("token_total")
   465	        if token_total and completion_metrics.get("generation_ms"):
   466	            try:
   467	                completion_metrics["time_per_token_ms"] = float(completion_metrics["generation_ms"]) / float(token_total)
   468	            except Exception:
   469	                pass
   470
   471	        await websocket.send_text(create_complete_message(
   472	            request_id=request_id,
   473	            metrics=completion_metrics,
   474	        ))
   475
   476	        # Log completion
   477	        if not cancelled:
   478	            log_request_completed(
   479	                request_id=request_id,
   480	                status_code=200,
   481	                total_duration_ms=int(total_time * 1000),
   482	                audio_duration_seconds=audio_duration,
   483	                rtf=rtf,
   484	                chunks_sent=chunks_sent,
   485	            )
   486	            metrics_request_completed(
   487	                status_code=200,
   488	                duration_seconds=total_time,
   489	                speaker=speaker,
   490	                stream=True,
   491	            )
   492	            record_rtf(rtf=rtf, speaker=speaker)
   493	            record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
   494	            record_chunks_sent(chunks=chunks_sent, speaker=speaker)
   495
   496	        # Close connection gracefully
   497	        await websocket.close(code=1000)
   498
   499	    except Exception as e:
   500	        logger.exception(f"WebSocket error: {e}")
   501	        log_request_failed(
   502	            request_id=request_id,
   503	            status_code=500,
   504	            error_code="WEBSOCKET_ERROR",
   505	            error_message=str(e),
   506	        )
   507	        metrics_request_failed(
   508	            status_code=500,
   509	            error_code="WEBSOCKET_ERROR",
   510	        )
   511
   512	        try:
   513	            await websocket.send_text(create_error_message(
   514	                code="INTERNAL_ERROR",
   515	                message=str(e),
   516	                request_id=request_id,
   517	            ))
   518	            await websocket.close(code=1011)
   519	        except Exception:
   520	            pass


sed -n '520,980p' scripts/stress_test_optimized_local.py


Output:
Chunk ID: 463a6f
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2933
Output:
            stream=stream,
        )
    except Exception as e:
        return RequestResult(
            success=False,
            status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error=str(e)[:200],
            speaker=speaker,
            text_length=len(text),
            stream=stream,
        )

async def run_level(
    base_url: str,
    num_requests: int,
    concurrency: int,
    text_category: str,
    chunking: bool,
    stream: bool,
    gpu_monitor: Optional[GPUMonitor] = None,
) -> LevelResult:
    mode = "streaming" if stream else "non-streaming"
    level = LevelResult(
        level_name=f"{concurrency}c-{text_category}-{mode}",
        concurrency=concurrency,
        total_requests=num_requests,
        mode=mode,
        text_category=text_category,
        chunking=chunking,
    )

    semaphore = asyncio.Semaphore(concurrency)

    if text_category == "mixed":
        texts = list(TEST_TEXTS.values())
    else:
        texts = [TEST_TEXTS.get(text_category, TEST_TEXTS["short"])]

    async def limited_request(i: int, client: httpx.AsyncClient) -> RequestResult:
        async with semaphore:
            text = texts[i % len(texts)]
            speaker = SPEAKERS[i % len(SPEAKERS)]
            return await make_request(
                client=client,
                base_url=base_url,
                text=text,
                speaker=speaker,
                chunking=chunking,
                stream=stream,
            )

    if gpu_monitor:
        gpu_monitor.snapshots.clear()

    start_time = time.time()
    limits = httpx.Limits(
        max_connections=concurrency + 10,
        max_keepalive_connections=concurrency,
    )
    async with httpx.AsyncClient(limits=limits) as client:
        tasks = [limited_request(i, client) for i in range(num_requests)]
        responses = await asyncio.gather(*tasks, return_exceptions=True)

    level.wall_time_seconds = time.time() - start_time

    for item in responses:
        if isinstance(item, RequestResult):
            level.results.append(item)
        else:
            level.results.append(
                RequestResult(
                    success=False,
                    status_code=0,
                    latency_ms=0,
                    error=str(item)[:200],
                    stream=stream,
                )
            )

    if gpu_monitor:
        level.gpu_summary = gpu_monitor.summary()

    return level

def print_level_report(level: LevelResult):
    print(f"\n{'=' * 72}")
    print(f"  {level.level_name.upper()}")
    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}, chunking={level.chunking}")
    print(f"{'=' * 72}")

    print(f"  Success:     {len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"  Wall time:   {level.wall_time_seconds:.2f}s")
    print(f"  Throughput:  {level.throughput_rps:.2f} req/s")
    print(f"  Audio total: {level.audio_duration_total:.2f}s")
    print(f"  Eff. RTF:    {level.effective_rtf:.3f}")

    if level.observed_inflight_max() is not None:
        print(f"  Peak inflight: {level.observed_inflight_max()}")

    latency = level.latency_stats
    if latency:
        print("\n  Latency (ms):")
        print(
            f"    min={latency['min']:.0f} avg={latency['avg']:.0f} "
            f"p50={latency['p50']:.0f} p95={latency['p95']:.0f} p99={latency['p99']:.0f} max={latency['max']:.0f}"
        )

    ttfb = level.ttfb_stats
    if ttfb and not level.mode.startswith("stream"):
        print("\n  Server TTFB (ms):")
        print(
            f"    min={ttfb['min']:.0f} avg={ttfb['avg']:.0f} "
            f"p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}"
        )

    if level.mode == "non-streaming":
        llm = level.timing_stats("generation_ms")
        if llm:
            print("\n  Generation (ms):")
            print(f"    min={llm['min']:.0f} avg={llm['avg']:.0f} p50={llm['p50']:.0f} p95={llm['p95']:.0f} max={llm['max']:.0f}")
        wall = level.timing_stats("llm_batch_wall_ms_total")
        if wall:
            print("\n  LLM batch wall (ms):")
            print(f"    min={wall['min']:.0f} avg={wall['avg']:.0f} p50={wall['p50']:.0f} p95={wall['p95']:.0f} max={wall['max']:.0f}")
        gpu = level.timing_stats("llm_batch_gpu_ms_total")
        if gpu:
            print("\n  LLM batch GPU (ms):")
            print(f"    min={gpu['min']:.0f} avg={gpu['avg']:.0f} p50={gpu['p50']:.0f} p95={gpu['p95']:.0f} max={gpu['max']:.0f}")
        parse = level.timing_stats("llm_parse_ms")
        if parse:
            print("\n  Parse (ms):")
            print(f"    avg={parse['avg']:.0f} p95={parse['p95']:.0f}")
        decode = level.timing_stats("llm_decode_wall_ms_total")
        if decode:
            print("\n  Decode (ms):")
            print(f"    avg={decode['avg']:.0f} p95={decode['p95']:.0f}")

        token_ms, token_count = level.timing_rate("generation_ms", "llm_token_total")
        if token_count:
            print(f"  Token timing: avg {token_ms / token_count * 1000:.3f} us/token")

    if level.gpu_summary and "memory_mb" in level.gpu_summary:
        gpu_summary = level.gpu_summary
        mem = gpu_summary["memory_mb"]
        util = gpu_summary["gpu_util_pct"]
        temp = gpu_summary["temperature_c"]
        print("\n  GPU:")
        print(f"    Memory: {mem['avg']:.0f}MB avg, {mem['max']:.0f}MB peak / {mem['total']:.0f}MB")
        print(f"    Util:   {util['avg']:.0f}% avg, {util['max']:.0f}% peak")
        print(f"    Temp:   {temp['avg']:.0f}C avg, {temp['max']:.0f}C peak")

    errors = level.error_summary()
    if errors:
        print("\n  Errors:")
        for key, count in sorted(errors.items(), key=lambda kv: -kv[1]):
            print(f"    [{count}x] {key}")

def print_summary_table(levels: List[LevelResult]):
    print(f"\n{'=' * 110}")
    print("  STRESS BENCHMARK SUMMARY")
    print(f"{'=' * 110}")
    header = f"{'Level':<26} {'OK%':>5} {'RPS':>7} {'p50':>7} {'p95':>7} {'GPU%':>5} {'GPUmem':>8} {'RTF':>6} {'Inflight':>8}"
    print(header)
    print("-" * 110)

    for level in levels:
        lat = level.latency_stats
        p50 = f"{lat.get('p50', 0):.0f}" if lat else "---"
        p95 = f"{lat.get('p95', 0):.0f}" if lat else "---"
        gpu_util = level.gpu_summary.get("gpu_util_pct", {}).get("avg", 0)
        gpu_mem = level.gpu_summary.get("memory_mb", {}).get("max", 0)
        inflight = level.observed_inflight_max() or 0
        print(
            f"{level.level_name:<26} "
            f"{level.success_rate:>4.0%} "
            f"{level.throughput_rps:>7.2f} "
            f"{p50:>7} "
            f"{p95:>7} "
            f"{gpu_util:>4.0f}% "
            f"{gpu_mem:>7.0f}MB "
            f"{level.effective_rtf:>6.3f} "
            f"{inflight:>8d}"
        )

    print("=" * 110)

def save_results(levels: List[LevelResult], path: str):
    payload = []
    for level in levels:
        payload.append({
            "level": level.level_name,
            "concurrency": level.concurrency,
            "mode": level.mode,
            "chunking": level.chunking,
            "text_category": level.text_category,
            "success_rate": level.success_rate,
            "throughput_rps": level.throughput_rps,
            "wall_time_s": level.wall_time_seconds,
            "latency_ms": level.latency_stats,
            "ttfb_ms": level.ttfb_stats,
            "effective_rtf": level.effective_rtf,
            "audio_total_seconds": level.audio_duration_total,
            "observed_inflight_max": level.observed_inflight_max(),
            "errors": level.error_summary(),
            "gpu": level.gpu_summary,
            "timing_averages": {
                "generation_ms": level.timing_stats("generation_ms"),
                "llm_batch_wall_ms_total": level.timing_stats("llm_batch_wall_ms_total"),
                "llm_batch_gpu_ms_total": level.timing_stats("llm_batch_gpu_ms_total"),
                "llm_parse_ms": level.timing_stats("llm_parse_ms"),
                "llm_decode_wall_ms_total": level.timing_stats("llm_decode_wall_ms_total"),
            },
            "requests": [
                {
                    "request_id": r.request_id,
                    "success": r.success,
                    "status_code": r.status_code,
                    "latency_ms": r.latency_ms,
                    "ttfb_ms": r.ttfb_ms,
                    "server_ttfb_ms": r.server_ttfb_ms,
                    "audio_bytes": r.audio_bytes,
                    "audio_seconds": r.audio_seconds,
                    "request_inflight": r.request_inflight,
                    "error": r.error,
                    "timing": r.timing,
                }
                for r in level.results
            ],
        })

    with open(path, "w") as fd:
        json.dump(payload, fd, indent=2)

async def run_stress_test(args):
    base_url = args.url

    try:
        async with httpx.AsyncClient() as client:
            response = await client.get(f"{base_url}/v1/tts/health", timeout=10)
            health = response.json()
            print(f"  Health: {health.get('status')}")
            print(f"  Version: {health.get('model_version')}")
            print(f"  GPU: {health.get('gpu_available')}")
    except Exception as e:
        print(f"ERROR: server not reachable at {base_url}: {e}")
        sys.exit(1)

    levels = [int(v) for v in args.levels.split(",") if v.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    mode_label = "streaming" if args.stream else "non-streaming"
    results: List[LevelResult] = []

    print(f"\nWarming up (1 request, non-streaming)...")
    warmup = await run_level(
        base_url=base_url,
        num_requests=1,
        concurrency=1,
        text_category=args.text,
        chunking=args.chunking,
        stream=False,
        gpu_monitor=gpu_monitor,
    )
    print(f"  warmup ok: {warmup.success_rate:.0%}")

    for conc in levels:
        num_requests = max(conc * 3, 10)
        num_requests = min(num_requests, 200)
        level = await run_level(
            base_url=base_url,
            num_requests=num_requests,
            concurrency=conc,
            text_category=args.text,
            chunking=args.chunking,
            stream=args.stream,
            gpu_monitor=gpu_monitor,
        )
        print_level_report(level)
        results.append(level)
        if conc != levels[-1]:
            print("  cooldown 3s...")
            await asyncio.sleep(3)

    gpu_monitor.stop()
    print_summary_table(results)

    if args.output:
        save_results(results, args.output)
        print(f"\nResults written to {args.output}")

def main():
    parser = argparse.ArgumentParser(description="Detailed local optimized-path TTS benchmark")
    parser.add_argument("--url", default=DEFAULT_URL, help=f"Server URL (default: {DEFAULT_URL})")
    parser.add_argument("--levels", default="1,5,10,20,50", help="Comma-separated concurrency levels")
    parser.add_argument("--text", default="short", choices=["short", "medium", "long", "mixed"], help="Input text category")
    parser.add_argument("--chunking", action="store_true", help="Enable server-side chunking", default=True)
    parser.add_argument("--no-chunking", action="store_false", dest="chunking", help="Disable chunking")
    parser.add_argument("--stream", action="store_true", help="Use streaming endpoint")
    parser.add_argument("--output", default="stress_test_optimized_local.json", help="Output JSON path")

    args = parser.parse_args()
    asyncio.run(run_stress_test(args))

if __name__ == "__main__":
    main()


I found the fastapi non-stream branch has a real indentation issue, and the stress script has a few metric parsing edge cases. I’ll patch both now and then run the benchmark script directly.


Output:
Chunk ID: a4c6de
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 4588
Output:
veena3modal/core/streaming_pipeline.py:52:def _detailed_timing_enabled() -> bool:
veena3modal/core/streaming_pipeline.py:53:    """Enable per-batch token/decode timing logs."""
veena3modal/core/streaming_pipeline.py:154:        request_id = f"slide-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
veena3modal/core/streaming_pipeline.py:158:            request_id=request_id,
veena3modal/core/streaming_pipeline.py:234:        request_id: Optional[str] = None,
veena3modal/core/streaming_pipeline.py:261:        emit_timing = _detailed_timing_enabled()
veena3modal/core/streaming_pipeline.py:265:                "request_id": request_id,
veena3modal/core/streaming_pipeline.py:308:        if request_id is None:
veena3modal/core/streaming_pipeline.py:310:            request_id = f"bicodec-stream-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
veena3modal/core/streaming_pipeline.py:311:            request_metrics["request_id"] = request_id
veena3modal/core/streaming_pipeline.py:332:            request_id=request_id,
veena3modal/core/streaming_pipeline.py:335:        # Track timing
veena3modal/core/streaming_pipeline.py:375:            if emit_timing:
veena3modal/core/streaming_pipeline.py:380:                        "request_id": request_id,
veena3modal/core/streaming_pipeline.py:420:                        if emit_timing and torch.cuda.is_available():
veena3modal/core/streaming_pipeline.py:422:                            decode_start_event = torch.cuda.Event(enable_timing=True)
veena3modal/core/streaming_pipeline.py:423:                            decode_end_event = torch.cuda.Event(enable_timing=True)
veena3modal/core/streaming_pipeline.py:436:                        if emit_timing and torch.cuda.is_available():
veena3modal/core/streaming_pipeline.py:481:                            if emit_timing:
veena3modal/core/streaming_pipeline.py:486:                                        "request_id": request_id,
veena3modal/core/streaming_pipeline.py:548:                "llm_request_id": request_id,
veena3modal/core/streaming_pipeline.py:581:        if emit_timing:
veena3modal/core/streaming_pipeline.py:594:        request_id: Optional[str] = None,
veena3modal/core/streaming_pipeline.py:635:        emit_timing = _detailed_timing_enabled()
veena3modal/core/streaming_pipeline.py:639:                "request_id": request_id,
veena3modal/core/streaming_pipeline.py:676:        if request_id is None:
veena3modal/core/streaming_pipeline.py:677:            request_id = f"bicodec-first-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
veena3modal/core/streaming_pipeline.py:678:            request_metrics["request_id"] = request_id
veena3modal/core/streaming_pipeline.py:688:            request_id=request_id,
veena3modal/core/streaming_pipeline.py:731:            if emit_timing:
veena3modal/core/streaming_pipeline.py:736:                        "request_id": request_id,
veena3modal/core/streaming_pipeline.py:761:                    if emit_timing and torch.cuda.is_available():
veena3modal/core/streaming_pipeline.py:763:                        decode_start_event = torch.cuda.Event(enable_timing=True)
veena3modal/core/streaming_pipeline.py:764:                        decode_end_event = torch.cuda.Event(enable_timing=True)
veena3modal/core/streaming_pipeline.py:787:                    if emit_timing and torch.cuda.is_available():
veena3modal/core/streaming_pipeline.py:817:                        if emit_timing:
veena3modal/core/streaming_pipeline.py:822:                                    "request_id": request_id,
veena3modal/core/streaming_pipeline.py:870:                if emit_timing:
veena3modal/core/streaming_pipeline.py:875:                            "request_id": request_id,
veena3modal/core/streaming_pipeline.py:905:                "request_id": request_id,
veena3modal/core/streaming_pipeline.py:957:        request_id: Optional[str] = None,
veena3modal/core/streaming_pipeline.py:997:        emit_timing = _detailed_timing_enabled()
veena3modal/core/streaming_pipeline.py:1001:                "request_id": request_id,
veena3modal/core/streaming_pipeline.py:1047:        if request_id is None:
veena3modal/core/streaming_pipeline.py:1048:            request_id = f"bicodec-cont-{uuid.uuid4().hex[:8]}-{int(time.time() * 1000000)}"
veena3modal/core/streaming_pipeline.py:1049:            request_metrics["request_id"] = request_id
veena3modal/core/streaming_pipeline.py:1059:            request_id=request_id,
veena3modal/core/streaming_pipeline.py:1100:            if emit_timing:
veena3modal/core/streaming_pipeline.py:1105:                        "request_id": request_id,
veena3modal/core/streaming_pipeline.py:1123:                    if emit_timing and torch.cuda.is_available():
veena3modal/core/streaming_pipeline.py:1125:                        decode_start_event = torch.cuda.Event(enable_timing=True)
veena3modal/core/streaming_pipeline.py:1126:                        decode_end_event = torch.cuda.Event(enable_timing=True)
veena3modal/core/streaming_pipeline.py:1150:                    if emit_timing and torch.cuda.is_available():
veena3modal/core/streaming_pipeline.py:1180:                        if emit_timing:
veena3modal/core/streaming_pipeline.py:1185:                                    "request_id": request_id,
veena3modal/core/streaming_pipeline.py:1233:                        if emit_timing:
veena3modal/core/streaming_pipeline.py:1238:                                    "request_id": request_id,
veena3modal/core/streaming_pipeline.py:1267:                "request_id": request_id,
veena3modal/processing/long_text_processor.py:153:        return_timing: bool = False,
veena3modal/processing/long_text_processor.py:154:        request_id: Optional[str] = None,
veena3modal/processing/long_text_processor.py:174:            Stitched audio bytes (int16 PCM) and optional timing metrics
veena3modal/processing/long_text_processor.py:176:        # Tracking variables for detailed timing aggregation
veena3modal/processing/long_text_processor.py:243:                request_id=request_id,
veena3modal/processing/long_text_processor.py:244:                return_timing=return_timing,
veena3modal/processing/long_text_processor.py:247:            if return_timing:
veena3modal/processing/long_text_processor.py:249:                    audio_bytes, chunk_timing = chunk_result
veena3modal/processing/long_text_processor.py:250:                    if isinstance(chunk_timing, dict):
veena3modal/processing/long_text_processor.py:251:                        emitted_chunk_metrics.append(chunk_timing)
veena3modal/processing/long_text_processor.py:252:                        llm_batch_wall_ms_values.append(int(chunk_timing.get("llm_wall_ms", 0)))
veena3modal/processing/long_text_processor.py:253:                        llm_batch_gpu_ms_values.append(int(chunk_timing.get("llm_gpu_batch_ms_total", 0)))
veena3modal/processing/long_text_processor.py:254:                        llm_decode_cpu_ms_values.append(int(chunk_timing.get("decode_cpu_ms", 0)))
veena3modal/processing/long_text_processor.py:255:                        llm_parse_ms_values.append(int(chunk_timing.get("parse_ms", 0)))
veena3modal/processing/long_text_processor.py:256:                        chunk_tokens = chunk_timing.get("token_batches", [])
veena3modal/processing/long_text_processor.py:308:        if return_timing:
veena3modal/processing/long_text_processor.py:311:                "request_id": request_id,
veena3modal/services/tts_runtime.py:29:_inflight_counter = 0
veena3modal/services/tts_runtime.py:30:_inflight_lock = Lock()
veena3modal/services/tts_runtime.py:33:def _timing_enabled() -> bool:
veena3modal/services/tts_runtime.py:34:    """Return True when detailed request timing should be emitted."""
veena3modal/services/tts_runtime.py:38:def _with_inflight_delta(delta: int) -> int:
veena3modal/services/tts_runtime.py:40:    global _inflight_counter
veena3modal/services/tts_runtime.py:41:    with _inflight_lock:
veena3modal/services/tts_runtime.py:42:        _inflight_counter = max(0, _inflight_counter + delta)
veena3modal/services/tts_runtime.py:43:        return _inflight_counter
veena3modal/services/tts_runtime.py:312:        request_id = f"warmup-{speaker_name}-{int(time.time() * 1000)}"
veena3modal/services/tts_runtime.py:318:            request_id=request_id,
veena3modal/services/tts_runtime.py:365:async def generate_speech(
veena3modal/services/tts_runtime.py:375:    request_id: Optional[str] = None,
veena3modal/services/tts_runtime.py:394:        metrics_dict: Dictionary with timing metrics
veena3modal/services/tts_runtime.py:404:        "request_id": request_id,
veena3modal/services/tts_runtime.py:413:    if request_id is None:
veena3modal/services/tts_runtime.py:415:        request_id = f"runtime-ns-{uuid.uuid4().hex[:10]}"
veena3modal/services/tts_runtime.py:416:        metrics["request_id"] = request_id
veena3modal/services/tts_runtime.py:418:    active_requests = _with_inflight_delta(1)
veena3modal/services/tts_runtime.py:421:    pipeline_timings = {}
veena3modal/services/tts_runtime.py:428:                "request_id": request_id,
veena3modal/services/tts_runtime.py:430:                "request_inflight": active_requests,
veena3modal/services/tts_runtime.py:449:        if _timing_enabled():
veena3modal/services/tts_runtime.py:450:            audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
veena3modal/services/tts_runtime.py:452:                return_timing=True,
veena3modal/services/tts_runtime.py:453:                request_id=request_id,
veena3modal/services/tts_runtime.py:458:                request_id=request_id,
veena3modal/services/tts_runtime.py:465:        if pipeline_timings:
veena3modal/services/tts_runtime.py:467:                "llm_token_total": pipeline_timings.get("token_total", 0),
veena3modal/services/tts_runtime.py:468:                "llm_batch_count": pipeline_timings.get("batch_count", 0),
veena3modal/services/tts_runtime.py:469:                "llm_batch_gpu_ms_total": pipeline_timings.get("llm_gpu_batch_ms_total", 0),
veena3modal/services/tts_runtime.py:470:                "llm_batch_gpu_ms_min": pipeline_timings.get("llm_gpu_batch_ms_min", 0),
veena3modal/services/tts_runtime.py:471:                "llm_batch_gpu_ms_max": pipeline_timings.get("llm_gpu_batch_ms_max", 0),
veena3modal/services/tts_runtime.py:472:                "llm_batch_gpu_ms_p50": pipeline_timings.get("llm_gpu_batch_ms_p50", 0),
veena3modal/services/tts_runtime.py:473:                "llm_decode_cpu_ms": pipeline_timings.get("decode_cpu_ms", 0),
veena3modal/services/tts_runtime.py:474:                "llm_parse_ms": pipeline_timings.get("parse_ms", 0),
veena3modal/services/tts_runtime.py:475:                "llm_wall_ms": pipeline_timings.get("llm_wall_ms", metrics["generation_ms"]),
veena3modal/services/tts_runtime.py:476:                "ttfb_ms": int(pipeline_timings.get("ttfb_ms", metrics["ttfb_ms"])),
veena3modal/services/tts_runtime.py:477:                "tokens_per_batch_min": pipeline_timings.get("tokens_per_batch_min"),
veena3modal/services/tts_runtime.py:478:                "tokens_per_batch_max": pipeline_timings.get("tokens_per_batch_max"),
veena3modal/services/tts_runtime.py:479:                "tokens_per_batch_p50": pipeline_timings.get("tokens_per_batch_p50"),
veena3modal/services/tts_runtime.py:516:        active_requests = _with_inflight_delta(-1)
veena3modal/services/tts_runtime.py:517:        if _timing_enabled():
veena3modal/services/tts_runtime.py:518:            metrics["request_inflight"] = active_requests
veena3modal/services/tts_runtime.py:524:        if _timing_enabled():
veena3modal/services/tts_runtime.py:526:                "tts_runtime_nonstream_inflight",
veena3modal/services/tts_runtime.py:528:                    "request_id": request_id,
veena3modal/services/tts_runtime.py:600:async def generate_speech_chunked(
veena3modal/services/tts_runtime.py:611:    request_id: Optional[str] = None,
veena3modal/services/tts_runtime.py:632:    if request_id is None:
veena3modal/services/tts_runtime.py:634:        request_id = f"runtime-chunk-{uuid.uuid4().hex[:10]}"
veena3modal/services/tts_runtime.py:636:        "request_id": request_id,
veena3modal/services/tts_runtime.py:646:    active_requests = _with_inflight_delta(1)
veena3modal/services/tts_runtime.py:650:        if _timing_enabled():
veena3modal/services/tts_runtime.py:655:                    "request_id": request_id,
veena3modal/services/tts_runtime.py:657:                    "request_inflight": active_requests,
veena3modal/services/tts_runtime.py:669:            if _timing_enabled():
veena3modal/services/tts_runtime.py:680:                    request_id=request_id,
veena3modal/services/tts_runtime.py:681:                    return_timing=True,
veena3modal/services/tts_runtime.py:684:                    audio_bytes, chunk_timing = chunk_result
veena3modal/services/tts_runtime.py:687:                    chunk_timing = {}
veena3modal/services/tts_runtime.py:699:                    request_id=request_id,
veena3modal/services/tts_runtime.py:701:                chunk_timing = {}
veena3modal/services/tts_runtime.py:713:                request_id=request_id,
veena3modal/services/tts_runtime.py:715:            if _timing_enabled():
veena3modal/services/tts_runtime.py:716:                audio_bytes, pipeline_timings = await runtime.pipeline.generate_speech_indic(
veena3modal/services/tts_runtime.py:718:                    return_timing=True,
veena3modal/services/tts_runtime.py:719:                    request_id=request_id,
veena3modal/services/tts_runtime.py:721:                chunk_timing = pipeline_timings
veena3modal/services/tts_runtime.py:725:                    request_id=request_id,
veena3modal/services/tts_runtime.py:727:                chunk_timing = {}
veena3modal/services/tts_runtime.py:733:        if chunk_timing:
veena3modal/services/tts_runtime.py:735:                "llm_token_total": chunk_timing.get("token_total", 0),
veena3modal/services/tts_runtime.py:736:                "llm_batch_count": chunk_timing.get("batch_count", 0),
veena3modal/services/tts_runtime.py:737:                "llm_batch_gpu_ms_total": chunk_timing.get("llm_gpu_batch_ms_total", 0),
veena3modal/services/tts_runtime.py:738:                "llm_batch_wall_ms_total": chunk_timing.get("llm_wall_ms", metrics["generation_ms"]),
veena3modal/services/tts_runtime.py:739:                "llm_parse_ms": chunk_timing.get("parse_ms", 0),
veena3modal/services/tts_runtime.py:740:                "llm_decode_cpu_ms": chunk_timing.get("decode_cpu_ms", 0),
veena3modal/services/tts_runtime.py:741:                "llm_batch_wall_ms_min": chunk_timing.get("llm_batch_wall_ms_min", 0),
veena3modal/services/tts_runtime.py:742:                "llm_batch_wall_ms_max": chunk_timing.get("llm_batch_wall_ms_max", 0),
veena3modal/services/tts_runtime.py:743:                "llm_batch_gpu_ms_min": chunk_timing.get("llm_batch_gpu_ms_min", 0),
veena3modal/services/tts_runtime.py:744:                "llm_batch_gpu_ms_max": chunk_timing.get("llm_batch_gpu_ms_max", 0),
veena3modal/services/tts_runtime.py:745:                "llm_batch_gpu_ms_p50": chunk_timing.get("llm_batch_gpu_ms_p50", 0),
veena3modal/services/tts_runtime.py:746:                "tokens_per_batch_min": chunk_timing.get("tokens_per_batch_min"),
veena3modal/services/tts_runtime.py:747:                "tokens_per_batch_max": chunk_timing.get("tokens_per_batch_max"),
veena3modal/services/tts_runtime.py:748:                "tokens_per_batch_p50": chunk_timing.get("tokens_per_batch_p50"),
veena3modal/services/tts_runtime.py:749:                "llm_batch_tokens": chunk_timing.get("token_batches", []),
veena3modal/services/tts_runtime.py:752:                metrics["chunks_processed"] = chunk_timing.get("chunks_processed", 0)
veena3modal/services/tts_runtime.py:776:        if _timing_enabled():
veena3modal/services/tts_runtime.py:778:            metrics["request_inflight"] = active_requests - 1
veena3modal/services/tts_runtime.py:786:        active_requests = _with_inflight_delta(-1)
veena3modal/services/tts_runtime.py:788:        metrics["request_inflight"] = active_requests
veena3modal/services/tts_runtime.py:794:async def generate_speech_streaming(
veena3modal/services/tts_runtime.py:804:    request_id: Optional[str] = None,
veena3modal/services/tts_runtime.py:828:        - Final yield: metrics_dict has final timing info
veena3modal/services/tts_runtime.py:838:    if request_id is None:
veena3modal/services/tts_runtime.py:840:        request_id = f"runtime-stream-{uuid.uuid4().hex[:10]}"
veena3modal/services/tts_runtime.py:850:        "request_id": request_id,
veena3modal/services/tts_runtime.py:860:        "request_id": request_id,
veena3modal/services/tts_runtime.py:870:    active_requests = _with_inflight_delta(1)
veena3modal/services/tts_runtime.py:871:    if _timing_enabled():
veena3modal/services/tts_runtime.py:877:                "request_inflight": active_requests,
veena3modal/services/tts_runtime.py:879:                "request_id": request_id,
veena3modal/services/tts_runtime.py:904:                request_id,
veena3modal/services/tts_runtime.py:940:                    request_id=request_id,
veena3modal/services/tts_runtime.py:953:                    request_id=request_id,
veena3modal/services/tts_runtime.py:970:        active_requests = _with_inflight_delta(-1)
veena3modal/services/tts_runtime.py:974:        if _timing_enabled():
veena3modal/services/tts_runtime.py:979:                    "request_id": request_id,
veena3modal/services/tts_runtime.py:980:                    "request_inflight": active_requests,
veena3modal/services/tts_runtime.py:999:    request_id: Optional[str] = None,
veena3modal/services/tts_runtime.py:1017:    chunk_metrics = request_metrics if request_metrics is not None else {"request_id": request_id}
veena3modal/services/tts_runtime.py:1018:    if "request_id" not in chunk_metrics:
veena3modal/services/tts_runtime.py:1019:        chunk_metrics["request_id"] = request_id
veena3modal/services/tts_runtime.py:1037:                request_id=request_id,
veena3modal/services/tts_runtime.py:1066:                    request_id=request_id,
veena3modal/services/tts_runtime.py:1089:                    request_id=request_id,


sed -n '1,260p' veena3modal/shared/logging.py