cat > /home/ubuntu/veenaModal/scripts/stress_test_optimized_local.py <<'PY'
#!/usr/bin/env python3
"""Detailed local stress benchmark for optimized TTS path.

This script is focused on collecting per-request GPU/LLM timing for local tuning:
- Request wall-clock latency and server reported TTFB
- GPU timing breakdown (LLM batch wall / GPU wall, parse, decode)
- Time per token and per batch
- Concurrency visibility from in-flight counters
- End-to-end throughput and RTF with GPU utilization snapshots

Usage:
    python scripts/stress_test_optimized_local.py
    python scripts/stress_test_optimized_local.py --levels 1,5,10,20 --concurrency-scaling both
    python scripts/stress_test_optimized_local.py --text-category long --output benchmarks.json
"""

import argparse
import asyncio
import json
import statistics
import subprocess
import sys
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

import httpx

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

TEST_TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This is a baseline quality sentence used for benchmarking local TTS latency, "
        "TTFB, and tokenization behavior under load."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast shadows over crowded streets, "
        "there lived a quiet linguist who believed every sentence carried a rhythm. "
        "She recorded conversations at dawn and dusk, trying to capture how meaning shifted with breath. "
        "By evening she would review them, matching sound to emotion, and every night she learned a new way to speak."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    """Background polling for quick GPU health snapshots."""

    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=3)

    def _poll_loop(self):
        while self._running:
            try:
                result = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True,
                    text=True,
                    timeout=5,
                )
                if result.returncode == 0 and result.stdout:
                    parts = [p.strip() for p in result.stdout.strip().split(",")]
                    if len(parts) >= 4:
                        self.snapshots.append(
                            GPUSnapshot(
                                timestamp=time.time(),
                                memory_used_mb=float(parts[0]),
                                memory_total_mb=float(parts[1]),
                                gpu_utilization_pct=float(parts[2]),
                                temperature_c=float(parts[3]),
                            )
                        )
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {
                "samples": 0,
                "error": "no GPU samples captured",
            }

        mem_used = [s.memory_used_mb for s in self.snapshots]
        gpu_util = [s.gpu_utilization_pct for s in self.snapshots]
        temp = [s.temperature_c for s in self.snapshots]

        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem_used),
                "max": max(mem_used),
                "avg": statistics.mean(mem_used),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(gpu_util),
                "max": max(gpu_util),
                "avg": statistics.mean(gpu_util),
            },
            "temperature_c": {
                "min": min(temp),
                "max": max(temp),
                "avg": statistics.mean(temp),
            },
        }

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float
    ttfb_ms: float = 0.0
    server_ttfb_ms: float = 0.0
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    request_id: str = ""
    error: Optional[str] = None
    speaker: str = ""
    text_length: int = 0
    stream: bool = False
    timing: Dict[str, Any] = field(default_factory=dict)
    request_inflight: Optional[int] = None

@dataclass
class LevelResult:
    level_name: str
    concurrency: int
    total_requests: int
    mode: str
    text_category: str
    chunking: bool
    results: List[RequestResult] = field(default_factory=list)
    wall_time_seconds: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0.0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_seconds if self.wall_time_seconds > 0 else 0.0

    @property
    def audio_duration_total(self) -> float:
        return sum(r.audio_seconds for r in self.successes)

    @property
    def effective_rtf(self) -> float:
        total_audio = self.audio_duration_total
        return self.wall_time_seconds / total_audio if total_audio > 0 else 0.0

    def _percentile(self, values: List[float], pct: float) -> float:
        if not values:
            return 0.0
        sorted_vals = sorted(values)
        idx = int(len(sorted_vals) * pct / 100)
        return sorted_vals[min(idx, len(sorted_vals) - 1)]

    @property
    def latency_stats(self) -> Dict[str, float]:
        lat = [r.latency_ms for r in self.successes]
        if not lat:
            return {}
        return {
            "min": min(lat),
            "avg": statistics.mean(lat),
            "p50": self._percentile(lat, 50),
            "p95": self._percentile(lat, 95),
            "p99": self._percentile(lat, 99),
            "max": max(lat),
        }

    @property
    def ttfb_stats(self) -> Dict[str, float]:
        vals = [r.server_ttfb_ms for r in self.successes if r.server_ttfb_ms >= 0]
        if not vals:
            return {}
        return {
            "min": min(vals),
            "avg": statistics.mean(vals),
            "p50": self._percentile(vals, 50),
            "p95": self._percentile(vals, 95),
            "max": max(vals),
        }

    def timing_stats(self, key: str) -> Dict[str, float]:
        values: List[float] = []
        for r in self.successes:
            v = r.timing.get(key)
            if isinstance(v, (int, float)):
                values.append(float(v))
        if not values:
            return {}
        return {
            "min": min(values),
            "avg": statistics.mean(values),
            "p50": self._percentile(values, 50),
            "p95": self._percentile(values, 95),
            "p99": self._percentile(values, 99),
            "max": max(values),
        }

    def timing_rate(self, numerator_key: str, denominator_key: str) -> Tuple[float, int]:
        num = 0.0
        den = 0
        for r in self.successes:
            n = r.timing.get(numerator_key)
            d = r.timing.get(denominator_key)
            if isinstance(n, (int, float)) and isinstance(d, (int, float)) and d > 0:
                num += float(n)
                den += int(d)
        if den == 0:
            return 0.0, 0
        return num / den, den

    def observed_inflight_max(self) -> Optional[int]:
        values = [r.request_inflight for r in self.successes if isinstance(r.request_inflight, int)]
        if not values:
            return None
        return max(values)

    def error_summary(self) -> Dict[str, int]:
        errors: Dict[str, int] = {}
        for r in self.failures:
            key = r.error or f"HTTP {r.status_code}"
            errors[key[:100]] = errors.get(key[:100], 0) + 1
        return errors

def _coerce_number(raw: str) -> Optional[float]:
    if raw is None:
        return None
    v = str(raw).strip()
    if v == "":
        return None
    try:
        if "." in v:
            return float(v)
        return float(int(v))
    except ValueError:
        try:
            return float(v)
        except ValueError:
            return None

def _normalize_time_ms(raw: Optional[str]) -> float:
    value = _coerce_number(raw)
    if value is None:
        return 0.0
    return float(value)

def _parse_perf_headers(headers: Any) -> Dict[str, Any]:
    """Parse server timing headers into numeric metric fields."""
    metrics: Dict[str, Any] = {}

    # Explicit compact payload.
    detail_payload = headers.get("x-perf-details") if hasattr(headers, "get") else None
    if detail_payload:
        try:
            parsed = json.loads(detail_payload)
            if isinstance(parsed, dict):
                metrics.update(parsed)
        except Exception:
            pass

    header_map = {
        "x-llm-token-total": "llm_token_total",
        "x-llm-batch-count": "llm_batch_count",
        "x-llm-batch-wall-ms": "llm_batch_wall_ms_total",
        "x-llm-batch-gpu-ms": "llm_batch_gpu_ms_total",
        "x-llm-batch-wall-ms-min": "llm_batch_wall_ms_min",
        "x-llm-batch-wall-ms-max": "llm_batch_wall_ms_max",
        "x-llm-batch-wall-ms-p50": "llm_batch_wall_ms_p50",
        "x-llm-batch-gpu-ms-min": "llm_batch_gpu_ms_min",
        "x-llm-batch-gpu-ms-max": "llm_batch_gpu_ms_max",
        "x-llm-batch-gpu-ms-p50": "llm_batch_gpu_ms_p50",
        "x-llm-decode-wall-ms": "llm_decode_wall_ms_total",
        "x-llm-decode-gpu-ms": "llm_decode_gpu_ms_total",
        "x-llm-decode-wall-ms-min": "llm_decode_wall_ms_min",
        "x-llm-decode-wall-ms-max": "llm_decode_wall_ms_max",
        "x-llm-decode-wall-ms-p50": "llm_decode_wall_ms_p50",
        "x-llm-decode-gpu-ms-min": "llm_decode_gpu_ms_min",
        "x-llm-decode-gpu-ms-max": "llm_decode_gpu_ms_max",
        "x-llm-decode-gpu-ms-p50": "llm_decode_gpu_ms_p50",
        "x-llm-decode-calls": "decode_calls",
        "x-llm-decode-cpu-ms": "llm_decode_cpu_ms",
        "x-llm-parse-ms": "llm_parse_ms",
        "x-llm-parse-avg-ms": "llm_parse_ms_avg",
        "x-llm-tokens-per-batch-min": "tokens_per_batch_min",
        "x-llm-tokens-per-batch-max": "tokens_per_batch_max",
        "x-llm-tokens-per-batch-p50": "tokens_per_batch_p50",
        "x-generation-ms": "generation_ms",
        "x-request-inflight": "request_inflight",
        "x-llm-text-chunked": "text_chunked",
        "x-llm-chunks-processed": "chunks_processed",
        "x-llm-time-per-token-ms": "llm_time_per_token_ms",
        "x-llm-time-per-batch-wall-ms": "llm_time_per_batch_wall_ms",
        "x-llm-time-per-batch-gpu-ms": "llm_time_per_batch_gpu_ms",
        "x-llm-parse-avg-ms": "llm_parse_ms_avg",
    }

    if hasattr(headers, "items"):
        for raw_key, raw_val in headers.items():
            lk = str(raw_key).lower()
            target_key = header_map.get(lk)
            if target_key is None:
                continue
            v = _coerce_number(raw_val)
            if v is not None:
                if target_key in {
                    "text_chunked",
                }:
                    metrics[target_key] = bool(v)
                else:
                    metrics[target_key] = int(v) if float(v).is_integer() else float(v)

    if "llm_token_total" in metrics and "generation_ms" in metrics and metrics["llm_token_total"]:
        try:
            metrics["llm_time_per_token_ms"] = float(metrics["generation_ms"]) / float(metrics["llm_token_total"])
        except Exception:
            pass

    if "llm_batch_count" in metrics and "generation_ms" in metrics and metrics["llm_batch_count"]:
        try:
            metrics["llm_time_per_batch_wall_ms"] = float(metrics["generation_ms"]) / float(metrics["llm_batch_count"])
        except Exception:
            pass

    if "llm_batch_count" in metrics and "llm_batch_gpu_ms_total" in metrics and metrics["llm_batch_count"]:
        try:
            metrics["llm_time_per_batch_gpu_ms"] = float(metrics["llm_batch_gpu_ms_total"]) / float(metrics["llm_batch_count"])
        except Exception:
            pass

    return metrics

async def make_request(
    client: httpx.AsyncClient,
    base_url: str,
    text: str,
    speaker: str,
    chunking: bool,
    stream: bool = False,
    timeout: float = 120.0,
) -> RequestResult:
    url = f"{base_url}{GENERATE_PATH}"
    payload = {
        "text": text,
        "speaker": speaker,
        "stream": stream,
        "chunking": chunking,
        "format": "wav",
    }

    start = time.time()
    ttfb_time = 0.0

    try:
        if stream:
            async with client.stream(
                "POST",
                url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            ) as response:
                first = True
                total_bytes = 0
                async for chunk in response.aiter_bytes():
                    if first:
                        ttfb_time = (time.time() - start) * 1000
                        first = False
                    total_bytes += len(chunk)

                latency = (time.time() - start) * 1000
                timing = _parse_perf_headers(response.headers)

                if response.status_code == 200:
                    pcm_bytes = max(0, total_bytes - 44)
                    audio_seconds = pcm_bytes / (16000 * 2)
                    return RequestResult(
                        success=True,
                        status_code=200,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        audio_bytes=total_bytes,
                        audio_seconds=audio_seconds,
                        request_id=response.headers.get("x-request-id", ""),
                        speaker=speaker,
                        text_length=len(text),
                        stream=True,
                        timing=timing,
                        request_inflight=timing.get("request_inflight"),
                        server_ttfb_ms=_normalize_time_ms(response.headers.get("x-ttfb-ms")),
                    )

                return RequestResult(
                    success=False,
                    status_code=response.status_code,
                    latency_ms=latency,
                    ttfb_ms=ttfb_time,
                    error=f"HTTP {response.status_code}",
                    speaker=speaker,
                    text_length=len(text),
                    stream=True,
                    timing=timing,
                    request_id=response.headers.get("x-request-id", ""),
                    request_inflight=timing.get("request_inflight"),
                )
        else:
            response = await client.post(
                url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            )
            latency = (time.time() - start) * 1000
            timing = _parse_perf_headers(response.headers)

            server_ttfb = _normalize_time_ms(response.headers.get("x-ttfb-ms"))
            audio_seconds = 0.0
            audio_sec_header = response.headers.get("x-audio-seconds")
            if audio_sec_header:
                parsed = _coerce_number(audio_sec_header)
                if parsed is not None:
                    audio_seconds = parsed
            elif response.status_code == 200:
                pcm_bytes = max(0, len(response.content) - 44)
                audio_seconds = pcm_bytes / (16000 * 2)

            if response.status_code == 200:
                return RequestResult(
                    success=True,
                    status_code=200,
                    latency_ms=latency,
                    ttfb_ms=latency,
                    server_ttfb_ms=server_ttfb,
                    audio_bytes=len(response.content),
                    audio_seconds=audio_seconds,
                    request_id=response.headers.get("x-request-id", ""),
                    speaker=speaker,
                    text_length=len(text),
                    stream=False,
                    timing=timing,
                    request_inflight=timing.get("request_inflight"),
                )

            return RequestResult(
                success=False,
                status_code=response.status_code,
                latency_ms=latency,
                ttfb_ms=latency,
                server_ttfb_ms=server_ttfb,
                audio_bytes=len(response.content),
                error=(response.text[:200] if response.text else f"HTTP {response.status_code}"),
                speaker=speaker,
                text_length=len(text),
                stream=False,
                timing=timing,
                request_id=response.headers.get("x-request-id", ""),
                request_inflight=timing.get("request_inflight"),
            )

    except httpx.TimeoutException:
        return RequestResult(
            success=False,
            status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error="TIMEOUT",
            speaker=speaker,
            text_length=len(text),
            stream=stream,
        )
    except Exception as e:
        return RequestResult(
            success=False,
            status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error=str(e)[:200],
            speaker=speaker,
            text_length=len(text),
            stream=stream,
        )

async def run_level(
    base_url: str,
    num_requests: int,
    concurrency: int,
    text_category: str,
    chunking: bool,
    stream: bool,
    gpu_monitor: Optional[GPUMonitor] = None,
) -> LevelResult:
    mode = "streaming" if stream else "non-streaming"
    level = LevelResult(
        level_name=f"{concurrency}c-{text_category}-{mode}",
        concurrency=concurrency,
        total_requests=num_requests,
        mode=mode,
        text_category=text_category,
        chunking=chunking,
    )

    semaphore = asyncio.Semaphore(concurrency)

    if text_category == "mixed":
        texts = list(TEST_TEXTS.values())
    else:
        texts = [TEST_TEXTS.get(text_category, TEST_TEXTS["short"])]

    async def limited_request(i: int, client: httpx.AsyncClient) -> RequestResult:
        async with semaphore:
            text = texts[i % len(texts)]
            speaker = SPEAKERS[i % len(SPEAKERS)]
            return await make_request(
                client=client,
                base_url=base_url,
                text=text,
                speaker=speaker,
                chunking=chunking,
                stream=stream,
            )

    if gpu_monitor:
        gpu_monitor.snapshots.clear()

    start_time = time.time()
    limits = httpx.Limits(
        max_connections=concurrency + 10,
        max_keepalive_connections=concurrency,
    )
    async with httpx.AsyncClient(limits=limits) as client:
        tasks = [limited_request(i, client) for i in range(num_requests)]
        responses = await asyncio.gather(*tasks, return_exceptions=True)

    level.wall_time_seconds = time.time() - start_time

    for item in responses:
        if isinstance(item, RequestResult):
            level.results.append(item)
        else:
            level.results.append(
                RequestResult(
                    success=False,
                    status_code=0,
                    latency_ms=0,
                    error=str(item)[:200],
                    stream=stream,
                )
            )

    if gpu_monitor:
        level.gpu_summary = gpu_monitor.summary()

    return level

def print_level_report(level: LevelResult):
    print(f"\n{'=' * 72}")
    print(f"  {level.level_name.upper()}")
    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}, chunking={level.chunking}")
    print(f"{'=' * 72}")

    print(f"  Success:     {len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"  Wall time:   {level.wall_time_seconds:.2f}s")
    print(f"  Throughput:  {level.throughput_rps:.2f} req/s")
    print(f"  Audio total: {level.audio_duration_total:.2f}s")
    print(f"  Eff. RTF:    {level.effective_rtf:.3f}")

    if level.observed_inflight_max() is not None:
        print(f"  Peak inflight: {level.observed_inflight_max()}")

    latency = level.latency_stats
    if latency:
        print("\n  Latency (ms):")
        print(
            f"    min={latency['min']:.0f} avg={latency['avg']:.0f} "
            f"p50={latency['p50']:.0f} p95={latency['p95']:.0f} p99={latency['p99']:.0f} max={latency['max']:.0f}"
        )

    ttfb = level.ttfb_stats
    if ttfb and not level.mode.startswith("stream"):
        print("\n  Server TTFB (ms):")
        print(
            f"    min={ttfb['min']:.0f} avg={ttfb['avg']:.0f} "
            f"p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}"
        )

    if level.mode == "non-streaming":
        llm = level.timing_stats("generation_ms")
        if llm:
            print("\n  Generation (ms):")
            print(f"    min={llm['min']:.0f} avg={llm['avg']:.0f} p50={llm['p50']:.0f} p95={llm['p95']:.0f} max={llm['max']:.0f}")
        wall = level.timing_stats("llm_batch_wall_ms_total")
        if wall:
            print("\n  LLM batch wall (ms):")
            print(f"    min={wall['min']:.0f} avg={wall['avg']:.0f} p50={wall['p50']:.0f} p95={wall['p95']:.0f} max={wall['max']:.0f}")
        gpu = level.timing_stats("llm_batch_gpu_ms_total")
        if gpu:
            print("\n  LLM batch GPU (ms):")
            print(f"    min={gpu['min']:.0f} avg={gpu['avg']:.0f} p50={gpu['p50']:.0f} p95={gpu['p95']:.0f} max={gpu['max']:.0f}")
        parse = level.timing_stats("llm_parse_ms")
        if parse:
            print("\n  Parse (ms):")
            print(f"    avg={parse['avg']:.0f} p95={parse['p95']:.0f}")
        decode = level.timing_stats("llm_decode_wall_ms_total")
        if decode:
            print("\n  Decode (ms):")
            print(f"    avg={decode['avg']:.0f} p95={decode['p95']:.0f}")

        token_ms, token_count = level.timing_rate("generation_ms", "llm_token_total")
        if token_count:
            print(f"  Token timing: avg {token_ms / token_count * 1000:.3f} us/token")

    if level.gpu_summary and "memory_mb" in level.gpu_summary:
        gpu_summary = level.gpu_summary
        mem = gpu_summary["memory_mb"]
        util = gpu_summary["gpu_util_pct"]
        temp = gpu_summary["temperature_c"]
        print("\n  GPU:")
        print(f"    Memory: {mem['avg']:.0f}MB avg, {mem['max']:.0f}MB peak / {mem['total']:.0f}MB")
        print(f"    Util:   {util['avg']:.0f}% avg, {util['max']:.0f}% peak")
        print(f"    Temp:   {temp['avg']:.0f}C avg, {temp['max']:.0f}C peak")

    errors = level.error_summary()
    if errors:
        print("\n  Errors:")
        for key, count in sorted(errors.items(), key=lambda kv: -kv[1]):
            print(f"    [{count}x] {key}")

def print_summary_table(levels: List[LevelResult]):
    print(f"\n{'=' * 110}")
    print("  STRESS BENCHMARK SUMMARY")
    print(f"{'=' * 110}")
    header = f"{'Level':<26} {'OK%':>5} {'RPS':>7} {'p50':>7} {'p95':>7} {'GPU%':>5} {'GPUmem':>8} {'RTF':>6} {'Inflight':>8}"
    print(header)
    print("-" * 110)

    for level in levels:
        lat = level.latency_stats
        p50 = f"{lat.get('p50', 0):.0f}" if lat else "---"
        p95 = f"{lat.get('p95', 0):.0f}" if lat else "---"
        gpu_util = level.gpu_summary.get("gpu_util_pct", {}).get("avg", 0)
        gpu_mem = level.gpu_summary.get("memory_mb", {}).get("max", 0)
        inflight = level.observed_inflight_max() or 0
        print(
            f"{level.level_name:<26} "
            f"{level.success_rate:>4.0%} "
            f"{level.throughput_rps:>7.2f} "
            f"{p50:>7} "
            f"{p95:>7} "
            f"{gpu_util:>4.0f}% "
            f"{gpu_mem:>7.0f}MB "
            f"{level.effective_rtf:>6.3f} "
            f"{inflight:>8d}"
        )

    print("=" * 110)

def save_results(levels: List[LevelResult], path: str):
    payload = []
    for level in levels:
        payload.append({
            "level": level.level_name,
            "concurrency": level.concurrency,
            "mode": level.mode,
            "chunking": level.chunking,
            "text_category": level.text_category,
            "success_rate": level.success_rate,
            "throughput_rps": level.throughput_rps,
            "wall_time_s": level.wall_time_seconds,
            "latency_ms": level.latency_stats,
            "ttfb_ms": level.ttfb_stats,
            "effective_rtf": level.effective_rtf,
            "audio_total_seconds": level.audio_duration_total,
            "observed_inflight_max": level.observed_inflight_max(),
            "errors": level.error_summary(),
            "gpu": level.gpu_summary,
            "timing_averages": {
                "generation_ms": level.timing_stats("generation_ms"),
                "llm_batch_wall_ms_total": level.timing_stats("llm_batch_wall_ms_total"),
                "llm_batch_gpu_ms_total": level.timing_stats("llm_batch_gpu_ms_total"),
                "llm_parse_ms": level.timing_stats("llm_parse_ms"),
                "llm_decode_wall_ms_total": level.timing_stats("llm_decode_wall_ms_total"),
            },
            "requests": [
                {
                    "request_id": r.request_id,
                    "success": r.success,
                    "status_code": r.status_code,
                    "latency_ms": r.latency_ms,
                    "ttfb_ms": r.ttfb_ms,
                    "server_ttfb_ms": r.server_ttfb_ms,
                    "audio_bytes": r.audio_bytes,
                    "audio_seconds": r.audio_seconds,
                    "request_inflight": r.request_inflight,
                    "error": r.error,
                    "timing": r.timing,
                }
                for r in level.results
            ],
        })

    with open(path, "w") as fd:
        json.dump(payload, fd, indent=2)

async def run_stress_test(args):
    base_url = args.url

    try:
        async with httpx.AsyncClient() as client:
            response = await client.get(f"{base_url}/v1/tts/health", timeout=10)
            health = response.json()
            print(f"  Health: {health.get('status')}")
            print(f"  Version: {health.get('model_version')}")
            print(f"  GPU: {health.get('gpu_available')}")
    except Exception as e:
        print(f"ERROR: server not reachable at {base_url}: {e}")
        sys.exit(1)

    levels = [int(v) for v in args.levels.split(",") if v.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    mode_label = "streaming" if args.stream else "non-streaming"
    results: List[LevelResult] = []

    print(f"\nWarming up (1 request, non-streaming)...")
    warmup = await run_level(
        base_url=base_url,
        num_requests=1,
        concurrency=1,
        text_category=args.text,
        chunking=args.chunking,
        stream=False,
        gpu_monitor=gpu_monitor,
    )
    print(f"  warmup ok: {warmup.success_rate:.0%}")

    for conc in levels:
        num_requests = max(conc * 3, 10)
        num_requests = min(num_requests, 200)
        level = await run_level(
            base_url=base_url,
            num_requests=num_requests,
            concurrency=conc,
            text_category=args.text,
            chunking=args.chunking,
            stream=args.stream,
            gpu_monitor=gpu_monitor,
        )
        print_level_report(level)
        results.append(level)
        if conc != levels[-1]:
            print("  cooldown 3s...")
            await asyncio.sleep(3)

    gpu_monitor.stop()
    print_summary_table(results)

    if args.output:
        save_results(results, args.output)
        print(f"\nResults written to {args.output}")

def main():
    parser = argparse.ArgumentParser(description="Detailed local optimized-path TTS benchmark")
    parser.add_argument("--url", default=DEFAULT_URL, help=f"Server URL (default: {DEFAULT_URL})")
    parser.add_argument("--levels", default="1,5,10,20,50", help="Comma-separated concurrency levels")
    parser.add_argument("--text", default="short", choices=["short", "medium", "long", "mixed"], help="Input text category")
    parser.add_argument("--chunking", action="store_true", help="Enable server-side chunking", default=True)
    parser.add_argument("--no-chunking", action="store_false", dest="chunking", help="Disable chunking")
    parser.add_argument("--stream", action="store_true", help="Use streaming endpoint")
    parser.add_argument("--output", default="stress_test_optimized_local.json", help="Output JSON path")

    args = parser.parse_args()
    asyncio.run(run_stress_test(args))

if __name__ == "__main__":
    main()
PY


Output:
Chunk ID: 4fd8cc
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 0
Output:


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


I added the optimized benchmark script with the new response-metrics parsing path. I’ll do a quick pass over the new file and wire-up details now, then provide exact run commands.


sed -n '1,240p' /home/ubuntu/veenaModal/scripts/stress_test_optimized_local.py


Output:
Chunk ID: 25be82
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 1936
Output:
#!/usr/bin/env python3
"""Detailed local stress benchmark for optimized TTS path.

This script is focused on collecting per-request GPU/LLM timing for local tuning:
- Request wall-clock latency and server reported TTFB
- GPU timing breakdown (LLM batch wall / GPU wall, parse, decode)
- Time per token and per batch
- Concurrency visibility from in-flight counters
- End-to-end throughput and RTF with GPU utilization snapshots

Usage:
    python scripts/stress_test_optimized_local.py
    python scripts/stress_test_optimized_local.py --levels 1,5,10,20 --concurrency-scaling both
    python scripts/stress_test_optimized_local.py --text-category long --output benchmarks.json
"""

import argparse
import asyncio
import json
import statistics
import subprocess
import sys
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

import httpx

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

TEST_TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This is a baseline quality sentence used for benchmarking local TTS latency, "
        "TTFB, and tokenization behavior under load."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast shadows over crowded streets, "
        "there lived a quiet linguist who believed every sentence carried a rhythm. "
        "She recorded conversations at dawn and dusk, trying to capture how meaning shifted with breath. "
        "By evening she would review them, matching sound to emotion, and every night she learned a new way to speak."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    """Background polling for quick GPU health snapshots."""

    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=3)

    def _poll_loop(self):
        while self._running:
            try:
                result = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True,
                    text=True,
                    timeout=5,
                )
                if result.returncode == 0 and result.stdout:
                    parts = [p.strip() for p in result.stdout.strip().split(",")]
                    if len(parts) >= 4:
                        self.snapshots.append(
                            GPUSnapshot(
                                timestamp=time.time(),
                                memory_used_mb=float(parts[0]),
                                memory_total_mb=float(parts[1]),
                                gpu_utilization_pct=float(parts[2]),
                                temperature_c=float(parts[3]),
                            )
                        )
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {
                "samples": 0,
                "error": "no GPU samples captured",
            }

        mem_used = [s.memory_used_mb for s in self.snapshots]
        gpu_util = [s.gpu_utilization_pct for s in self.snapshots]
        temp = [s.temperature_c for s in self.snapshots]

        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem_used),
                "max": max(mem_used),
                "avg": statistics.mean(mem_used),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(gpu_util),
                "max": max(gpu_util),
                "avg": statistics.mean(gpu_util),
            },
            "temperature_c": {
                "min": min(temp),
                "max": max(temp),
                "avg": statistics.mean(temp),
            },
        }

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float
    ttfb_ms: float = 0.0
    server_ttfb_ms: float = 0.0
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    request_id: str = ""
    error: Optional[str] = None
    speaker: str = ""
    text_length: int = 0
    stream: bool = False
    timing: Dict[str, Any] = field(default_factory=dict)
    request_inflight: Optional[int] = None

@dataclass
class LevelResult:
    level_name: str
    concurrency: int
    total_requests: int
    mode: str
    text_category: str
    chunking: bool
    results: List[RequestResult] = field(default_factory=list)
    wall_time_seconds: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0.0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_seconds if self.wall_time_seconds > 0 else 0.0

    @property
    def audio_duration_total(self) -> float:
        return sum(r.audio_seconds for r in self.successes)

    @property
    def effective_rtf(self) -> float:
        total_audio = self.audio_duration_total
        return self.wall_time_seconds / total_audio if total_audio > 0 else 0.0

    def _percentile(self, values: List[float], pct: float) -> float:
        if not values:
            return 0.0
        sorted_vals = sorted(values)
        idx = int(len(sorted_vals) * pct / 100)
        return sorted_vals[min(idx, len(sorted_vals) - 1)]

    @property
    def latency_stats(self) -> Dict[str, float]:
        lat = [r.latency_ms for r in self.successes]
        if not lat:
            return {}
        return {
            "min": min(lat),
            "avg": statistics.mean(lat),
            "p50": self._percentile(lat, 50),
            "p95": self._percentile(lat, 95),
            "p99": self._percentile(lat, 99),
            "max": max(lat),
        }

    @property
    def ttfb_stats(self) -> Dict[str, float]:
        vals = [r.server_ttfb_ms for r in self.successes if r.server_ttfb_ms >= 0]
        if not vals:
            return {}
        return {
            "min": min(vals),
            "avg": statistics.mean(vals),
            "p50": self._percentile(vals, 50),
            "p95": self._percentile(vals, 95),
            "max": max(vals),
        }

    def timing_stats(self, key: str) -> Dict[str, float]:
        values: List[float] = []
        for r in self.successes:
            v = r.timing.get(key)
            if isinstance(v, (int, float)):
                values.append(float(v))
        if not values:
            return {}
        return {
            "min": min(values),


Summary: auto


sed -n '240,520p' /home/ubuntu/veenaModal/scripts/stress_test_optimized_local.py


Output:
Chunk ID: 1b437a
Wall time: 0.0538 seconds
Process exited with code 0
Original token count: 2651
Output:
            "min": min(values),
            "avg": statistics.mean(values),
            "p50": self._percentile(values, 50),
            "p95": self._percentile(values, 95),
            "p99": self._percentile(values, 99),
            "max": max(values),
        }

    def timing_rate(self, numerator_key: str, denominator_key: str) -> Tuple[float, int]:
        num = 0.0
        den = 0
        for r in self.successes:
            n = r.timing.get(numerator_key)
            d = r.timing.get(denominator_key)
            if isinstance(n, (int, float)) and isinstance(d, (int, float)) and d > 0:
                num += float(n)
                den += int(d)
        if den == 0:
            return 0.0, 0
        return num / den, den

    def observed_inflight_max(self) -> Optional[int]:
        values = [r.request_inflight for r in self.successes if isinstance(r.request_inflight, int)]
        if not values:
            return None
        return max(values)

    def error_summary(self) -> Dict[str, int]:
        errors: Dict[str, int] = {}
        for r in self.failures:
            key = r.error or f"HTTP {r.status_code}"
            errors[key[:100]] = errors.get(key[:100], 0) + 1
        return errors

def _coerce_number(raw: str) -> Optional[float]:
    if raw is None:
        return None
    v = str(raw).strip()
    if v == "":
        return None
    try:
        if "." in v:
            return float(v)
        return float(int(v))
    except ValueError:
        try:
            return float(v)
        except ValueError:
            return None

def _normalize_time_ms(raw: Optional[str]) -> float:
    value = _coerce_number(raw)
    if value is None:
        return 0.0
    return float(value)

def _parse_perf_headers(headers: Any) -> Dict[str, Any]:
    """Parse server timing headers into numeric metric fields."""
    metrics: Dict[str, Any] = {}

    # Explicit compact payload.
    detail_payload = headers.get("x-perf-details") if hasattr(headers, "get") else None
    if detail_payload:
        try:
            parsed = json.loads(detail_payload)
            if isinstance(parsed, dict):
                metrics.update(parsed)
        except Exception:
            pass

    header_map = {
        "x-llm-token-total": "llm_token_total",
        "x-llm-batch-count": "llm_batch_count",
        "x-llm-batch-wall-ms": "llm_batch_wall_ms_total",
        "x-llm-batch-gpu-ms": "llm_batch_gpu_ms_total",
        "x-llm-batch-wall-ms-min": "llm_batch_wall_ms_min",
        "x-llm-batch-wall-ms-max": "llm_batch_wall_ms_max",
        "x-llm-batch-wall-ms-p50": "llm_batch_wall_ms_p50",
        "x-llm-batch-gpu-ms-min": "llm_batch_gpu_ms_min",
        "x-llm-batch-gpu-ms-max": "llm_batch_gpu_ms_max",
        "x-llm-batch-gpu-ms-p50": "llm_batch_gpu_ms_p50",
        "x-llm-decode-wall-ms": "llm_decode_wall_ms_total",
        "x-llm-decode-gpu-ms": "llm_decode_gpu_ms_total",
        "x-llm-decode-wall-ms-min": "llm_decode_wall_ms_min",
        "x-llm-decode-wall-ms-max": "llm_decode_wall_ms_max",
        "x-llm-decode-wall-ms-p50": "llm_decode_wall_ms_p50",
        "x-llm-decode-gpu-ms-min": "llm_decode_gpu_ms_min",
        "x-llm-decode-gpu-ms-max": "llm_decode_gpu_ms_max",
        "x-llm-decode-gpu-ms-p50": "llm_decode_gpu_ms_p50",
        "x-llm-decode-calls": "decode_calls",
        "x-llm-decode-cpu-ms": "llm_decode_cpu_ms",
        "x-llm-parse-ms": "llm_parse_ms",
        "x-llm-parse-avg-ms": "llm_parse_ms_avg",
        "x-llm-tokens-per-batch-min": "tokens_per_batch_min",
        "x-llm-tokens-per-batch-max": "tokens_per_batch_max",
        "x-llm-tokens-per-batch-p50": "tokens_per_batch_p50",
        "x-generation-ms": "generation_ms",
        "x-request-inflight": "request_inflight",
        "x-llm-text-chunked": "text_chunked",
        "x-llm-chunks-processed": "chunks_processed",
        "x-llm-time-per-token-ms": "llm_time_per_token_ms",
        "x-llm-time-per-batch-wall-ms": "llm_time_per_batch_wall_ms",
        "x-llm-time-per-batch-gpu-ms": "llm_time_per_batch_gpu_ms",
        "x-llm-parse-avg-ms": "llm_parse_ms_avg",
    }

    if hasattr(headers, "items"):
        for raw_key, raw_val in headers.items():
            lk = str(raw_key).lower()
            target_key = header_map.get(lk)
            if target_key is None:
                continue
            v = _coerce_number(raw_val)
            if v is not None:
                if target_key in {
                    "text_chunked",
                }:
                    metrics[target_key] = bool(v)
                else:
                    metrics[target_key] = int(v) if float(v).is_integer() else float(v)

    if "llm_token_total" in metrics and "generation_ms" in metrics and metrics["llm_token_total"]:
        try:
            metrics["llm_time_per_token_ms"] = float(metrics["generation_ms"]) / float(metrics["llm_token_total"])
        except Exception:
            pass

    if "llm_batch_count" in metrics and "generation_ms" in metrics and metrics["llm_batch_count"]:
        try:
            metrics["llm_time_per_batch_wall_ms"] = float(metrics["generation_ms"]) / float(metrics["llm_batch_count"])
        except Exception:
            pass

    if "llm_batch_count" in metrics and "llm_batch_gpu_ms_total" in metrics and metrics["llm_batch_count"]:
        try:
            metrics["llm_time_per_batch_gpu_ms"] = float(metrics["llm_batch_gpu_ms_total"]) / float(metrics["llm_batch_count"])
        except Exception:
            pass

    return metrics

async def make_request(
    client: httpx.AsyncClient,
    base_url: str,
    text: str,
    speaker: str,
    chunking: bool,
    stream: bool = False,
    timeout: float = 120.0,
) -> RequestResult:
    url = f"{base_url}{GENERATE_PATH}"
    payload = {
        "text": text,
        "speaker": speaker,
        "stream": stream,
        "chunking": chunking,
        "format": "wav",
    }

    start = time.time()
    ttfb_time = 0.0

    try:
        if stream:
            async with client.stream(
                "POST",
                url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            ) as response:
                first = True
                total_bytes = 0
                async for chunk in response.aiter_bytes():
                    if first:
                        ttfb_time = (time.time() - start) * 1000
                        first = False
                    total_bytes += len(chunk)

                latency = (time.time() - start) * 1000
                timing = _parse_perf_headers(response.headers)

                if response.status_code == 200:
                    pcm_bytes = max(0, total_bytes - 44)
                    audio_seconds = pcm_bytes / (16000 * 2)
                    return RequestResult(
                        success=True,
                        status_code=200,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        audio_bytes=total_bytes,
                        audio_seconds=audio_seconds,
                        request_id=response.headers.get("x-request-id", ""),
                        speaker=speaker,
                        text_length=len(text),
                        stream=True,
                        timing=timing,
                        request_inflight=timing.get("request_inflight"),
                        server_ttfb_ms=_normalize_time_ms(response.headers.get("x-ttfb-ms")),
                    )

                return RequestResult(
                    success=False,
                    status_code=response.status_code,
                    latency_ms=latency,
                    ttfb_ms=ttfb_time,
                    error=f"HTTP {response.status_code}",
                    speaker=speaker,
                    text_length=len(text),
                    stream=True,
                    timing=timing,
                    request_id=response.headers.get("x-request-id", ""),
                    request_inflight=timing.get("request_inflight"),
                )
        else:
            response = await client.post(
                url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            )
            latency = (time.time() - start) * 1000
            timing = _parse_perf_headers(response.headers)

            server_ttfb = _normalize_time_ms(response.headers.get("x-ttfb-ms"))
            audio_seconds = 0.0
            audio_sec_header = response.headers.get("x-audio-seconds")
            if audio_sec_header:
                parsed = _coerce_number(audio_sec_header)
                if parsed is not None:
                    audio_seconds = parsed
            elif response.status_code == 200:
                pcm_bytes = max(0, len(response.content) - 44)
                audio_seconds = pcm_bytes / (16000 * 2)

            if response.status_code == 200:
                return RequestResult(
                    success=True,
                    status_code=200,
                    latency_ms=latency,
                    ttfb_ms=latency,
                    server_ttfb_ms=server_ttfb,
                    audio_bytes=len(response.content),
                    audio_seconds=audio_seconds,
                    request_id=response.headers.get("x-request-id", ""),
                    speaker=speaker,
                    text_length=len(text),
                    stream=False,
                    timing=timing,
                    request_inflight=timing.get("request_inflight"),
                )

            return RequestResult(
                success=False,
                status_code=response.status_code,
                latency_ms=latency,
                ttfb_ms=latency,
                server_ttfb_ms=server_ttfb,
                audio_bytes=len(response.content),
                error=(response.text[:200] if response.text else f"HTTP {response.status_code}"),
                speaker=speaker,
                text_length=len(text),
                stream=False,
                timing=timing,
                request_id=response.headers.get("x-request-id", ""),
                request_inflight=timing.get("request_inflight"),
            )

    except httpx.TimeoutException:
        return RequestResult(
            success=False,
            status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error="TIMEOUT",
            speaker=speaker,
            text_length=len(text),
            stream=stream,


sed -n '520,920p' /home/ubuntu/veenaModal/scripts/stress_test_optimized_local.py


Output:
Chunk ID: 38446f
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2933
Output:
            stream=stream,
        )
    except Exception as e:
        return RequestResult(
            success=False,
            status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error=str(e)[:200],
            speaker=speaker,
            text_length=len(text),
            stream=stream,
        )

async def run_level(
    base_url: str,
    num_requests: int,
    concurrency: int,
    text_category: str,
    chunking: bool,
    stream: bool,
    gpu_monitor: Optional[GPUMonitor] = None,
) -> LevelResult:
    mode = "streaming" if stream else "non-streaming"
    level = LevelResult(
        level_name=f"{concurrency}c-{text_category}-{mode}",
        concurrency=concurrency,
        total_requests=num_requests,
        mode=mode,
        text_category=text_category,
        chunking=chunking,
    )

    semaphore = asyncio.Semaphore(concurrency)

    if text_category == "mixed":
        texts = list(TEST_TEXTS.values())
    else:
        texts = [TEST_TEXTS.get(text_category, TEST_TEXTS["short"])]

    async def limited_request(i: int, client: httpx.AsyncClient) -> RequestResult:
        async with semaphore:
            text = texts[i % len(texts)]
            speaker = SPEAKERS[i % len(SPEAKERS)]
            return await make_request(
                client=client,
                base_url=base_url,
                text=text,
                speaker=speaker,
                chunking=chunking,
                stream=stream,
            )

    if gpu_monitor:
        gpu_monitor.snapshots.clear()

    start_time = time.time()
    limits = httpx.Limits(
        max_connections=concurrency + 10,
        max_keepalive_connections=concurrency,
    )
    async with httpx.AsyncClient(limits=limits) as client:
        tasks = [limited_request(i, client) for i in range(num_requests)]
        responses = await asyncio.gather(*tasks, return_exceptions=True)

    level.wall_time_seconds = time.time() - start_time

    for item in responses:
        if isinstance(item, RequestResult):
            level.results.append(item)
        else:
            level.results.append(
                RequestResult(
                    success=False,
                    status_code=0,
                    latency_ms=0,
                    error=str(item)[:200],
                    stream=stream,
                )
            )

    if gpu_monitor:
        level.gpu_summary = gpu_monitor.summary()

    return level

def print_level_report(level: LevelResult):
    print(f"\n{'=' * 72}")
    print(f"  {level.level_name.upper()}")
    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}, chunking={level.chunking}")
    print(f"{'=' * 72}")

    print(f"  Success:     {len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"  Wall time:   {level.wall_time_seconds:.2f}s")
    print(f"  Throughput:  {level.throughput_rps:.2f} req/s")
    print(f"  Audio total: {level.audio_duration_total:.2f}s")
    print(f"  Eff. RTF:    {level.effective_rtf:.3f}")

    if level.observed_inflight_max() is not None:
        print(f"  Peak inflight: {level.observed_inflight_max()}")

    latency = level.latency_stats
    if latency:
        print("\n  Latency (ms):")
        print(
            f"    min={latency['min']:.0f} avg={latency['avg']:.0f} "
            f"p50={latency['p50']:.0f} p95={latency['p95']:.0f} p99={latency['p99']:.0f} max={latency['max']:.0f}"
        )

    ttfb = level.ttfb_stats
    if ttfb and not level.mode.startswith("stream"):
        print("\n  Server TTFB (ms):")
        print(
            f"    min={ttfb['min']:.0f} avg={ttfb['avg']:.0f} "
            f"p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}"
        )

    if level.mode == "non-streaming":
        llm = level.timing_stats("generation_ms")
        if llm:
            print("\n  Generation (ms):")
            print(f"    min={llm['min']:.0f} avg={llm['avg']:.0f} p50={llm['p50']:.0f} p95={llm['p95']:.0f} max={llm['max']:.0f}")
        wall = level.timing_stats("llm_batch_wall_ms_total")
        if wall:
            print("\n  LLM batch wall (ms):")
            print(f"    min={wall['min']:.0f} avg={wall['avg']:.0f} p50={wall['p50']:.0f} p95={wall['p95']:.0f} max={wall['max']:.0f}")
        gpu = level.timing_stats("llm_batch_gpu_ms_total")
        if gpu:
            print("\n  LLM batch GPU (ms):")
            print(f"    min={gpu['min']:.0f} avg={gpu['avg']:.0f} p50={gpu['p50']:.0f} p95={gpu['p95']:.0f} max={gpu['max']:.0f}")
        parse = level.timing_stats("llm_parse_ms")
        if parse:
            print("\n  Parse (ms):")
            print(f"    avg={parse['avg']:.0f} p95={parse['p95']:.0f}")
        decode = level.timing_stats("llm_decode_wall_ms_total")
        if decode:
            print("\n  Decode (ms):")
            print(f"    avg={decode['avg']:.0f} p95={decode['p95']:.0f}")

        token_ms, token_count = level.timing_rate("generation_ms", "llm_token_total")
        if token_count:
            print(f"  Token timing: avg {token_ms / token_count * 1000:.3f} us/token")

    if level.gpu_summary and "memory_mb" in level.gpu_summary:
        gpu_summary = level.gpu_summary
        mem = gpu_summary["memory_mb"]
        util = gpu_summary["gpu_util_pct"]
        temp = gpu_summary["temperature_c"]
        print("\n  GPU:")
        print(f"    Memory: {mem['avg']:.0f}MB avg, {mem['max']:.0f}MB peak / {mem['total']:.0f}MB")
        print(f"    Util:   {util['avg']:.0f}% avg, {util['max']:.0f}% peak")
        print(f"    Temp:   {temp['avg']:.0f}C avg, {temp['max']:.0f}C peak")

    errors = level.error_summary()
    if errors:
        print("\n  Errors:")
        for key, count in sorted(errors.items(), key=lambda kv: -kv[1]):
            print(f"    [{count}x] {key}")

def print_summary_table(levels: List[LevelResult]):
    print(f"\n{'=' * 110}")
    print("  STRESS BENCHMARK SUMMARY")
    print(f"{'=' * 110}")
    header = f"{'Level':<26} {'OK%':>5} {'RPS':>7} {'p50':>7} {'p95':>7} {'GPU%':>5} {'GPUmem':>8} {'RTF':>6} {'Inflight':>8}"
    print(header)
    print("-" * 110)

    for level in levels:
        lat = level.latency_stats
        p50 = f"{lat.get('p50', 0):.0f}" if lat else "---"
        p95 = f"{lat.get('p95', 0):.0f}" if lat else "---"
        gpu_util = level.gpu_summary.get("gpu_util_pct", {}).get("avg", 0)
        gpu_mem = level.gpu_summary.get("memory_mb", {}).get("max", 0)
        inflight = level.observed_inflight_max() or 0
        print(
            f"{level.level_name:<26} "
            f"{level.success_rate:>4.0%} "
            f"{level.throughput_rps:>7.2f} "
            f"{p50:>7} "
            f"{p95:>7} "
            f"{gpu_util:>4.0f}% "
            f"{gpu_mem:>7.0f}MB "
            f"{level.effective_rtf:>6.3f} "
            f"{inflight:>8d}"
        )

    print("=" * 110)

def save_results(levels: List[LevelResult], path: str):
    payload = []
    for level in levels:
        payload.append({
            "level": level.level_name,
            "concurrency": level.concurrency,
            "mode": level.mode,
            "chunking": level.chunking,
            "text_category": level.text_category,
            "success_rate": level.success_rate,
            "throughput_rps": level.throughput_rps,
            "wall_time_s": level.wall_time_seconds,
            "latency_ms": level.latency_stats,
            "ttfb_ms": level.ttfb_stats,
            "effective_rtf": level.effective_rtf,
            "audio_total_seconds": level.audio_duration_total,
            "observed_inflight_max": level.observed_inflight_max(),
            "errors": level.error_summary(),
            "gpu": level.gpu_summary,
            "timing_averages": {
                "generation_ms": level.timing_stats("generation_ms"),
                "llm_batch_wall_ms_total": level.timing_stats("llm_batch_wall_ms_total"),
                "llm_batch_gpu_ms_total": level.timing_stats("llm_batch_gpu_ms_total"),
                "llm_parse_ms": level.timing_stats("llm_parse_ms"),
                "llm_decode_wall_ms_total": level.timing_stats("llm_decode_wall_ms_total"),
            },
            "requests": [
                {
                    "request_id": r.request_id,
                    "success": r.success,
                    "status_code": r.status_code,
                    "latency_ms": r.latency_ms,
                    "ttfb_ms": r.ttfb_ms,
                    "server_ttfb_ms": r.server_ttfb_ms,
                    "audio_bytes": r.audio_bytes,
                    "audio_seconds": r.audio_seconds,
                    "request_inflight": r.request_inflight,
                    "error": r.error,
                    "timing": r.timing,
                }
                for r in level.results
            ],
        })

    with open(path, "w") as fd:
        json.dump(payload, fd, indent=2)

async def run_stress_test(args):
    base_url = args.url

    try:
        async with httpx.AsyncClient() as client:
            response = await client.get(f"{base_url}/v1/tts/health", timeout=10)
            health = response.json()
            print(f"  Health: {health.get('status')}")
            print(f"  Version: {health.get('model_version')}")
            print(f"  GPU: {health.get('gpu_available')}")
    except Exception as e:
        print(f"ERROR: server not reachable at {base_url}: {e}")
        sys.exit(1)

    levels = [int(v) for v in args.levels.split(",") if v.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    mode_label = "streaming" if args.stream else "non-streaming"
    results: List[LevelResult] = []

    print(f"\nWarming up (1 request, non-streaming)...")
    warmup = await run_level(
        base_url=base_url,
        num_requests=1,
        concurrency=1,
        text_category=args.text,
        chunking=args.chunking,
        stream=False,
        gpu_monitor=gpu_monitor,
    )
    print(f"  warmup ok: {warmup.success_rate:.0%}")

    for conc in levels:
        num_requests = max(conc * 3, 10)
        num_requests = min(num_requests, 200)
        level = await run_level(
            base_url=base_url,
            num_requests=num_requests,
            concurrency=conc,
            text_category=args.text,
            chunking=args.chunking,
            stream=args.stream,
            gpu_monitor=gpu_monitor,
        )
        print_level_report(level)
        results.append(level)
        if conc != levels[-1]:
            print("  cooldown 3s...")
            await asyncio.sleep(3)

    gpu_monitor.stop()
    print_summary_table(results)

    if args.output:
        save_results(results, args.output)
        print(f"\nResults written to {args.output}")

def main():
    parser = argparse.ArgumentParser(description="Detailed local optimized-path TTS benchmark")
    parser.add_argument("--url", default=DEFAULT_URL, help=f"Server URL (default: {DEFAULT_URL})")
    parser.add_argument("--levels", default="1,5,10,20,50", help="Comma-separated concurrency levels")
    parser.add_argument("--text", default="short", choices=["short", "medium", "long", "mixed"], help="Input text category")
    parser.add_argument("--chunking", action="store_true", help="Enable server-side chunking", default=True)
    parser.add_argument("--no-chunking", action="store_false", dest="chunking", help="Disable chunking")
    parser.add_argument("--stream", action="store_true", help="Use streaming endpoint")
    parser.add_argument("--output", default="stress_test_optimized_local.json", help="Output JSON path")

    args = parser.parse_args()
    asyncio.run(run_stress_test(args))

if __name__ == "__main__":
    main()


sed -n '1,160p' /home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py


Output:
Chunk ID: 78a069
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 1462
Output:
"""
FastAPI ASGI app factory for Modal deployment.

Phase 1: expose `/v1/tts/generate` and `/v1/tts/health` while reusing existing
Veena3 inference/streaming code from the Django project.

Phase 2+: extract shared code into `veena3modal/shared/` and remove Django coupling.
"""

# NOTE: Intentionally NOT using `from __future__ import annotations` here
# to ensure FastAPI recognizes Request type annotations properly

import time
import uuid
import json
from typing import Dict, Optional

# Structured logging (JSON formatted, request-scoped)
from veena3modal.shared.logging import (
    get_logger,
    set_request_context,
    clear_request_context,
    log_request_received,
    log_first_audio_emitted,
    log_request_completed,
    log_request_failed,
)

# Prometheus metrics
from veena3modal.shared.metrics import (
    record_request_received as metrics_request_received,
    record_request_completed as metrics_request_completed,
    record_request_failed as metrics_request_failed,
    record_ttfb,
    record_rtf,
    record_audio_duration,
    record_chunks_sent,
    get_metrics_text,
    get_content_type as get_metrics_content_type,
)

# Auth and rate limiting
from veena3modal.api.auth import (
    get_api_validator,
    extract_api_key,
    hash_api_key,
)
from veena3modal.api.rate_limiter import get_rate_limiter
from veena3modal.api.error_handlers import (
    ErrorCode,
    create_error_response,
    get_error_status,
    is_gpu_fault,
    handle_gpu_fault,
    FeatureFlags,
)

logger = get_logger(__name__)

# Sentence storage (fire-and-forget, non-blocking)
# Imported at module level for singleton access
_sentence_store = None

def _get_sentence_store():
    """Lazy load sentence store singleton."""
    global _sentence_store
    if _sentence_store is None:
        from veena3modal.services.sentence_store import get_sentence_store
        _sentence_store = get_sentence_store()
    return _sentence_store

def _build_perf_headers(metrics: Dict[str, object]) -> Dict[str, str]:
    """Build compact timing headers for benchmark tooling."""
    perf: Dict[str, str] = {}
    if not metrics:
        return perf

    def _pick(*keys: str):
        for key in keys:
            if key in metrics and metrics[key] is not None:
                return metrics[key]
        return None

    def _add(name: str, value: object):
        if value is None:
            return
        if isinstance(value, bool):
            perf[name] = str(value).lower()
        else:
            perf[name] = str(value)

    llm_token_total = _pick("llm_token_total", "token_total")
    llm_batch_count = _pick("llm_batch_count", "batch_count")
    generation_ms = _pick("generation_ms") or _pick("total_ms")
    llm_batch_wall_total = _pick("llm_batch_wall_ms_total", "llm_wall_ms", "total_ms")
    llm_batch_gpu_total = _pick("llm_batch_gpu_ms_total", "llm_gpu_batch_ms_total")
    llm_decode_wall_total = _pick("llm_decode_wall_ms_total")
    llm_decode_gpu_total = _pick("llm_decode_gpu_ms_total")
    llm_parse_ms = _pick("llm_parse_ms")
    decode_cpu_ms = _pick("llm_decode_cpu_ms")
    parse_ms = _pick("parse_ms")

    _add("X-LLM-Token-Total", llm_token_total)
    _add("X-LLM-Batch-Count", llm_batch_count)
    _add("X-LLM-Batch-Wall-Ms", llm_batch_wall_total)
    _add("X-LLM-Batch-GPU-Ms", llm_batch_gpu_total)
    _add("X-LLM-Batch-Wall-Ms-Min", _pick("llm_batch_wall_ms_min"))
    _add("X-LLM-Batch-Wall-Ms-Max", _pick("llm_batch_wall_ms_max"))
    _add("X-LLM-Batch-Wall-Ms-P50", _pick("llm_batch_wall_ms_p50"))
    _add("X-LLM-Batch-GPU-Ms-Min", _pick("llm_batch_gpu_ms_min"))
    _add("X-LLM-Batch-GPU-Ms-Max", _pick("llm_batch_gpu_ms_max"))
    _add("X-LLM-Batch-GPU-Ms-P50", _pick("llm_batch_gpu_ms_p50"))
    _add("X-LLM-Decode-Wall-Ms", llm_decode_wall_total)
    _add("X-LLM-Decode-GPU-Ms", llm_decode_gpu_total)
    _add("X-LLM-Decode-Wall-Ms-Min", _pick("llm_decode_wall_ms_min"))
    _add("X-LLM-Decode-Wall-Ms-Max", _pick("llm_decode_wall_ms_max"))
    _add("X-LLM-Decode-Wall-Ms-P50", _pick("llm_decode_wall_ms_p50"))
    _add("X-LLM-Decode-GPU-Ms-Min", _pick("llm_decode_gpu_ms_min"))
    _add("X-LLM-Decode-GPU-Ms-Max", _pick("llm_decode_gpu_ms_max"))
    _add("X-LLM-Decode-GPU-Ms-P50", _pick("llm_decode_gpu_ms_p50"))
    _add("X-LLM-Decode-Calls", _pick("decode_calls"))
    _add("X-LLM-Decode-CPU-Ms", decode_cpu_ms)
    _add("X-LLM-Parse-Ms", llm_parse_ms or parse_ms)
    _add("X-LLM-Parse-Avg-Ms", _pick("llm_parse_ms_avg"))
    _add("X-LLM-Tokens-Per-Batch-Min", _pick("tokens_per_batch_min"))
    _add("X-LLM-Tokens-Per-Batch-Max", _pick("tokens_per_batch_max"))
    _add("X-LLM-Tokens-Per-Batch-P50", _pick("tokens_per_batch_p50"))
    _add("X-Generation-Ms", generation_ms)
    _add("X-LLM-Generation-Ms", _pick("llm_wall_ms", generation_ms))
    _add("X-Request-Inflight", _pick("request_inflight", "active_requests"))
    _add("X-LLM-Text-Chunked", _pick("text_chunked"))
    _add("X-LLM-Chunks-Processed", _pick("chunks_processed"))
    _add("X-LLM-SR-Applied", _pick("sr_applied"))

    if llm_token_total and generation_ms:
        try:
            perf["X-LLM-Time-Per-Token-Ms"] = f"{float(generation_ms) / float(llm_token_total):.3f}"
        except Exception:
            pass

    if llm_batch_count:
        try:
            perf["X-LLM-Time-Per-Batch-Wall-Ms"] = f"{float(generation_ms or 0) / float(llm_batch_count):.3f}"
        except Exception:
            pass
        try:
            perf["X-LLM-Time-Per-Batch-GPU-Ms"] = f"{float(llm_batch_gpu_total or 0) / float(llm_batch_count):.3f}"
        except Exception:
            pass

    # Include compact JSON snapshot for downstream benchmark tooling.
    detail_keys = [
        "request_id",
        "speaker",
        "text_length",
        "llm_token_total",
        "llm_batch_count",
        "llm_batch_wall_ms_total",


The current TTS application here is an implementation in Modal labs. Which has autoscaling and other rules self scaling rules and everything suitable for my target audience. Now, i need you to create a local entry point or implementation point. then we will plan for optimising the pipeline, then we will discuss on stress testing, then we will discuss on scaling laws post optimisations locally. @veena3modal/app.py

the above part is performed by a different agent. and benchmarks are made. Now can you do the benchmarks on this newly optimized path to understand the concurrency and stress testing.

Now i need you to capture detailed time logs for each request happening in the GPU. like what happens with 1req, time per LLM token, time per batch decoding, and everything else. detailed time about where we are spending each minute in the GPU per generation.

and when you have the details for per request. you can obviously add more logging to get details of concurrency. so you understand what's the bottleneck for the peak concurrency. and then we will decide on effective methods in optimising this TTS engine.

Warning: apply_patch was requested via exec_command. Use the apply_patch tool instead of exec_command.

<permissions instructions>
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `danger-full-access`: No filesystem sandboxing - all commands are permitted. Network access is enabled.
Approval policy is currently never. Do not provide the `sandbox_permissions` for any reason, commands will be rejected.
</permissions instructions>

<collaboration_mode># Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.
</collaboration_mode>

# AGENTS.md instructions for /home/ubuntu/veenaModal

<INSTRUCTIONS>
## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.
</INSTRUCTIONS>

<environment_context>
  <cwd>/home/ubuntu/veenaModal</cwd>
  <shell>bash</shell>
</environment_context>


I’ll quickly validate and finalize the benchmark path end-to-end: patch the benchmark parser/flags if needed, ensure endpoint metric headers are coherent, then run the local stress test against the optimized app and return request-level plus aggregated GPU/concurrency timings.


sed -n '1,260p' veena3modal/api/fastapi_app.py


sed -n '1,280p' veena3modal/api/websocket_handler.py


sed -n '1,280p' scripts/stress_test_optimized_local.py


Output:
Chunk ID: a81b75
Wall time: 0.0516 seconds
Process exited with code 0
Original token count: 2236
Output:
"""
FastAPI ASGI app factory for Modal deployment.

Phase 1: expose `/v1/tts/generate` and `/v1/tts/health` while reusing existing
Veena3 inference/streaming code from the Django project.

Phase 2+: extract shared code into `veena3modal/shared/` and remove Django coupling.
"""

# NOTE: Intentionally NOT using `from __future__ import annotations` here
# to ensure FastAPI recognizes Request type annotations properly

import time
import uuid
import json
from typing import Dict, Optional

# Structured logging (JSON formatted, request-scoped)
from veena3modal.shared.logging import (
    get_logger,
    set_request_context,
    clear_request_context,
    log_request_received,
    log_first_audio_emitted,
    log_request_completed,
    log_request_failed,
)

# Prometheus metrics
from veena3modal.shared.metrics import (
    record_request_received as metrics_request_received,
    record_request_completed as metrics_request_completed,
    record_request_failed as metrics_request_failed,
    record_ttfb,
    record_rtf,
    record_audio_duration,
    record_chunks_sent,
    get_metrics_text,
    get_content_type as get_metrics_content_type,
)

# Auth and rate limiting
from veena3modal.api.auth import (
    get_api_validator,
    extract_api_key,
    hash_api_key,
)
from veena3modal.api.rate_limiter import get_rate_limiter
from veena3modal.api.error_handlers import (
    ErrorCode,
    create_error_response,
    get_error_status,
    is_gpu_fault,
    handle_gpu_fault,
    FeatureFlags,
)

logger = get_logger(__name__)

# Sentence storage (fire-and-forget, non-blocking)
# Imported at module level for singleton access
_sentence_store = None

def _get_sentence_store():
    """Lazy load sentence store singleton."""
    global _sentence_store
    if _sentence_store is None:
        from veena3modal.services.sentence_store import get_sentence_store
        _sentence_store = get_sentence_store()
    return _sentence_store

def _build_perf_headers(metrics: Dict[str, object]) -> Dict[str, str]:
    """Build compact timing headers for benchmark tooling."""
    perf: Dict[str, str] = {}
    if not metrics:
        return perf

    def _pick(*keys: str):
        for key in keys:
            if key in metrics and metrics[key] is not None:
                return metrics[key]
        return None

    def _add(name: str, value: object):
        if value is None:
            return
        if isinstance(value, bool):
            perf[name] = str(value).lower()
        else:
            perf[name] = str(value)

    llm_token_total = _pick("llm_token_total", "token_total")
    llm_batch_count = _pick("llm_batch_count", "batch_count")
    generation_ms = _pick("generation_ms") or _pick("total_ms")
    llm_batch_wall_total = _pick("llm_batch_wall_ms_total", "llm_wall_ms", "total_ms")
    llm_batch_gpu_total = _pick("llm_batch_gpu_ms_total", "llm_gpu_batch_ms_total")
    llm_decode_wall_total = _pick("llm_decode_wall_ms_total")
    llm_decode_gpu_total = _pick("llm_decode_gpu_ms_total")
    llm_parse_ms = _pick("llm_parse_ms")
    decode_cpu_ms = _pick("llm_decode_cpu_ms")
    parse_ms = _pick("parse_ms")

    _add("X-LLM-Token-Total", llm_token_total)
    _add("X-LLM-Batch-Count", llm_batch_count)
    _add("X-LLM-Batch-Wall-Ms", llm_batch_wall_total)
    _add("X-LLM-Batch-GPU-Ms", llm_batch_gpu_total)
    _add("X-LLM-Batch-Wall-Ms-Min", _pick("llm_batch_wall_ms_min"))
    _add("X-LLM-Batch-Wall-Ms-Max", _pick("llm_batch_wall_ms_max"))
    _add("X-LLM-Batch-Wall-Ms-P50", _pick("llm_batch_wall_ms_p50"))
    _add("X-LLM-Batch-GPU-Ms-Min", _pick("llm_batch_gpu_ms_min"))
    _add("X-LLM-Batch-GPU-Ms-Max", _pick("llm_batch_gpu_ms_max"))
    _add("X-LLM-Batch-GPU-Ms-P50", _pick("llm_batch_gpu_ms_p50"))
    _add("X-LLM-Decode-Wall-Ms", llm_decode_wall_total)
    _add("X-LLM-Decode-GPU-Ms", llm_decode_gpu_total)
    _add("X-LLM-Decode-Wall-Ms-Min", _pick("llm_decode_wall_ms_min"))
    _add("X-LLM-Decode-Wall-Ms-Max", _pick("llm_decode_wall_ms_max"))
    _add("X-LLM-Decode-Wall-Ms-P50", _pick("llm_decode_wall_ms_p50"))
    _add("X-LLM-Decode-GPU-Ms-Min", _pick("llm_decode_gpu_ms_min"))
    _add("X-LLM-Decode-GPU-Ms-Max", _pick("llm_decode_gpu_ms_max"))
    _add("X-LLM-Decode-GPU-Ms-P50", _pick("llm_decode_gpu_ms_p50"))
    _add("X-LLM-Decode-Calls", _pick("decode_calls"))
    _add("X-LLM-Decode-CPU-Ms", decode_cpu_ms)
    _add("X-LLM-Parse-Ms", llm_parse_ms or parse_ms)
    _add("X-LLM-Parse-Avg-Ms", _pick("llm_parse_ms_avg"))
    _add("X-LLM-Tokens-Per-Batch-Min", _pick("tokens_per_batch_min"))
    _add("X-LLM-Tokens-Per-Batch-Max", _pick("tokens_per_batch_max"))
    _add("X-LLM-Tokens-Per-Batch-P50", _pick("tokens_per_batch_p50"))
    _add("X-Generation-Ms", generation_ms)
    _add("X-LLM-Generation-Ms", _pick("llm_wall_ms", generation_ms))
    _add("X-Request-Inflight", _pick("request_inflight", "active_requests"))
    _add("X-LLM-Text-Chunked", _pick("text_chunked"))
    _add("X-LLM-Chunks-Processed", _pick("chunks_processed"))
    _add("X-LLM-SR-Applied", _pick("sr_applied"))

    if llm_token_total and generation_ms:
        try:
            perf["X-LLM-Time-Per-Token-Ms"] = f"{float(generation_ms) / float(llm_token_total):.3f}"
        except Exception:
            pass

    if llm_batch_count:
        try:
            perf["X-LLM-Time-Per-Batch-Wall-Ms"] = f"{float(generation_ms or 0) / float(llm_batch_count):.3f}"
        except Exception:
            pass
        try:
            perf["X-LLM-Time-Per-Batch-GPU-Ms"] = f"{float(llm_batch_gpu_total or 0) / float(llm_batch_count):.3f}"
        except Exception:
            pass

    # Include compact JSON snapshot for downstream benchmark tooling.
    detail_keys = [
        "request_id",
        "speaker",
        "text_length",
        "llm_token_total",
        "llm_batch_count",
        "llm_batch_wall_ms_total",
        "llm_batch_gpu_ms_total",
        "llm_batch_wall_ms_min",
        "llm_batch_wall_ms_max",
        "llm_batch_wall_ms_p50",
        "llm_batch_gpu_ms_min",
        "llm_batch_gpu_ms_max",
        "llm_batch_gpu_ms_p50",
        "llm_decode_cpu_ms",
        "llm_decode_wall_ms_total",
        "llm_decode_gpu_ms_total",
        "llm_decode_wall_ms_min",
        "llm_decode_wall_ms_max",
        "llm_decode_wall_ms_p50",
        "llm_decode_gpu_ms_min",
        "llm_decode_gpu_ms_max",
        "llm_decode_gpu_ms_p50",
        "decode_calls",
        "decode_cpu_ms",
        "parse_ms",
        "llm_parse_ms",
        "llm_parse_ms_avg",
        "llm_batch_tokens",
        "tokens_per_batch_min",
        "tokens_per_batch_max",
        "tokens_per_batch_p50",
        "generation_ms",
        "audio_duration_seconds",
        "request_inflight",
        "active_requests",
        "chunks_processed",
        "text_chunked",
        "sr_applied",
    ]
    perf["X-Perf-Details"] = json.dumps(
        {k: metrics[k] for k in detail_keys if k in metrics},
        separators=(",", ":"),
    )

    return perf

# Version info (will be set dynamically after model load)
_APP_VERSION = "0.1.0"
_MODEL_VERSION: Optional[str] = None
_startup_time: Optional[float] = None

def get_model_version() -> str:
    """Return the current model version, or 'not_loaded' if not initialized."""
    return _MODEL_VERSION or "not_loaded"

def set_model_version(version: str) -> None:
    """Set the model version after loading."""
    global _MODEL_VERSION
    _MODEL_VERSION = version

def create_app():
    """
    Factory function to create FastAPI app.

    Import inside factory so local tooling can import this module without FastAPI installed.
    """
    from fastapi import FastAPI, HTTPException, Request
    from fastapi.responses import JSONResponse, Response

    global _startup_time
    _startup_time = time.time()

    app = FastAPI(
        title="Veena3 TTS (Modal)",
        version=_APP_VERSION,
        description="High-quality multilingual Text-to-Speech API with true streaming support.",
    )

    @app.get("/v1/tts/health")
    async def tts_health():
        """
        Health check endpoint.

        Returns:
            - status: "healthy" | "degraded" | "unhealthy"
            - model_loaded: whether TTS model is initialized
            - model_version: version string of loaded model
            - uptime_seconds: time since app startup
            - gpu_available: whether GPU is detected (best-effort check)
        """
        # Check model status via runtime
        from veena3modal.services import tts_runtime
        model_loaded = tts_runtime.is_initialized()

        # Use runtime's model version if available
        runtime = tts_runtime.get_runtime()
        model_version = runtime.model_version if runtime else get_model_version()

        # Check GPU availability (best-effort, non-blocking)
        gpu_available = False
        try:
            import torch
            gpu_available = torch.cuda.is_available()


Output:
Chunk ID: 51da70
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 2128
Output:
"""
WebSocket handler for TTS streaming.

Provides bidirectional WebSocket communication for TTS:
- Client sends JSON request with text and parameters
- Server streams audio chunks as binary frames
- Client can send control messages (cancel, ping)

Protocol:
1. Client connects to /v1/tts/ws
2. Client sends JSON: {"text": "...", "speaker": "...", ...}
3. Server sends binary audio chunks
4. Server sends JSON: {"event": "complete", "metrics": {...}}
5. Connection closes

Control messages:
- {"event": "cancel"} - Stop generation
- {"event": "ping"} - Keep-alive (server responds with pong)
"""

import asyncio
import json
import time
import uuid
from typing import Optional, Dict, Any
from dataclasses import dataclass
from enum import Enum

from veena3modal.shared.logging import (
    get_logger,
    set_request_context,
    clear_request_context,
    log_request_received,
    log_first_audio_emitted,
    log_request_completed,
    log_request_failed,
)
from veena3modal.shared.metrics import (
    record_request_received as metrics_request_received,
    record_request_completed as metrics_request_completed,
    record_request_failed as metrics_request_failed,
    record_ttfb,
    record_rtf,
    record_audio_duration,
    record_chunks_sent,
)

logger = get_logger(__name__)

class WSMessageType(str, Enum):
    """WebSocket message types."""
    # Client -> Server
    REQUEST = "request"
    CANCEL = "cancel"
    PING = "ping"

    # Server -> Client
    AUDIO_CHUNK = "audio_chunk"  # Binary frame
    HEADER = "header"  # WAV header info
    PROGRESS = "progress"  # Progress update
    COMPLETE = "complete"  # Generation complete
    ERROR = "error"  # Error occurred
    PONG = "pong"  # Response to ping

@dataclass
class WSRequest:
    """Parsed WebSocket TTS request."""
    text: str
    speaker: str = "male1"
    temperature: float = 0.8
    top_k: int = 50
    top_p: float = 1.0
    max_tokens: int = 4096
    repetition_penalty: float = 1.05
    seed: Optional[int] = None
    chunking: bool = False
    normalize: bool = True
    format: str = "wav"
    sample_rate: int = 16000

    @classmethod
    def from_dict(cls, data: Dict[str, Any]) -> "WSRequest":
        """Parse request from JSON dict."""
        return cls(
            text=data.get("text", ""),
            speaker=data.get("speaker", "male1"),
            temperature=data.get("temperature", 0.8),
            top_k=data.get("top_k", 50),
            top_p=data.get("top_p", 1.0),
            max_tokens=data.get("max_tokens", 4096),
            repetition_penalty=data.get("repetition_penalty", 1.05),
            seed=data.get("seed"),
            chunking=data.get("chunking", False),
            normalize=data.get("normalize", True),
            format=data.get("format", "wav"),
            sample_rate=data.get("sample_rate", 16000),
        )

    def validate(self) -> Optional[str]:
        """Validate request, return error message if invalid."""
        if not self.text or not self.text.strip():
            return "Text is required and cannot be empty"
        if len(self.text) > 50000:
            return f"Text too long: {len(self.text)} chars (max 50000)"
        if self.format != "wav":
            return f"WebSocket streaming only supports WAV format (got {self.format})"
        return None

def create_error_message(code: str, message: str, request_id: str) -> str:
    """Create JSON error message."""
    return json.dumps({
        "event": WSMessageType.ERROR.value,
        "error": {
            "code": code,
            "message": message,
            "request_id": request_id,
        }
    })

def create_header_message(
    request_id: str,
    sample_rate: int,
    model_version: str,
) -> str:
    """Create JSON header message (sent before audio)."""
    return json.dumps({
        "event": WSMessageType.HEADER.value,
        "request_id": request_id,
        "sample_rate": sample_rate,
        "format": "wav",
        "model_version": model_version,
    })

def create_progress_message(
    chunks_sent: int,
    bytes_sent: int,
    elapsed_ms: int,
) -> str:
    """Create JSON progress message."""
    return json.dumps({
        "event": WSMessageType.PROGRESS.value,
        "chunks_sent": chunks_sent,
        "bytes_sent": bytes_sent,
        "elapsed_ms": elapsed_ms,
    })

def create_complete_message(
    request_id: str,
    metrics: Dict[str, Any],
) -> str:
    """Create JSON completion message."""
    return json.dumps({
        "event": WSMessageType.COMPLETE.value,
        "request_id": request_id,
        "metrics": metrics,
    })

_METRIC_FORWARD_KEYS = {
    "ttfb_ms",
    "generation_ms",
    "audio_duration_seconds",
    "audio_chunks",
    "total_bytes",
    "chunks_sent",
    "request_inflight",
    "active_requests",
    "llm_token_total",
    "llm_batch_count",
    "llm_batch_wall_ms_total",
    "llm_batch_wall_ms_min",
    "llm_batch_wall_ms_max",
    "llm_batch_wall_ms_p50",
    "llm_batch_gpu_ms_total",
    "llm_batch_gpu_ms_min",
    "llm_batch_gpu_ms_max",
    "llm_batch_gpu_ms_p50",
    "llm_decode_cpu_ms",
    "llm_decode_wall_ms_total",
    "llm_decode_wall_ms_min",
    "llm_decode_wall_ms_max",
    "llm_decode_wall_ms_p50",
    "llm_decode_gpu_ms_total",
    "llm_decode_gpu_ms_min",
    "llm_decode_gpu_ms_max",
    "llm_decode_gpu_ms_p50",
    "decode_calls",
    "parse_ms",
    "llm_parse_ms",
    "llm_parse_ms_avg",
    "tokens_per_batch_min",
    "tokens_per_batch_max",
    "tokens_per_batch_p50",
    "llm_batch_tokens",
    "generated_tokens",
    "semantic_tokens",
    "global_tokens",
    "text_chunked",
    "concurrency_mode",
    "captured_global_count",
    "chunks_processed",
}

async def handle_websocket_tts(websocket, api_key: Optional[str] = None):
    """
    Handle WebSocket TTS streaming connection.

    This is the main handler called from FastAPI WebSocket route.

    Args:
        websocket: FastAPI WebSocket instance
        api_key: Optional API key for authentication
    """
    from veena3modal.api.auth import get_api_validator, hash_api_key
    from veena3modal.api.rate_limiter import get_rate_limiter
    from veena3modal.api.error_handlers import FeatureFlags
    from veena3modal.api.schemas import resolve_speaker_name
    from veena3modal.services import tts_runtime

    request_id = str(uuid.uuid4())
    set_request_context(request_id=request_id)

    # Accept the WebSocket connection
    await websocket.accept()
    logger.info(f"WebSocket connection accepted: {request_id}")

    # Track cancellation
    cancelled = False

    try:
        # === AUTH CHECK (if enabled) ===
        if FeatureFlags.is_auth_enabled() and api_key:
            validator = get_api_validator()
            key_hash = hash_api_key(api_key)
            auth_result = validator.validate(key_hash)

            if not auth_result.is_valid:
                await websocket.send_text(create_error_message(
                    code=auth_result.error_code or "AUTH_FAILED",
                    message=auth_result.error_message or "Authentication failed",
                    request_id=request_id,
                ))
                await websocket.close(code=1008)  # Policy violation
                return

            # Rate limit check
            if FeatureFlags.is_rate_limiting_enabled():
                rate_limiter = get_rate_limiter()
                allowed, remaining, reset_after = rate_limiter.check(key_hash)
                if not allowed:
                    await websocket.send_text(create_error_message(
                        code="RATE_LIMIT_EXCEEDED",
                        message=f"Rate limit exceeded. Retry after {int(reset_after) + 1} seconds.",
                        request_id=request_id,
                    ))
                    await websocket.close(code=1008)
                    return

        # === WAIT FOR REQUEST ===
        try:
            # Wait for client to send request (with timeout)
            raw_message = await asyncio.wait_for(
                websocket.receive_text(),
                timeout=30.0
            )
            message = json.loads(raw_message)
        except asyncio.TimeoutError:
            await websocket.send_text(create_error_message(
                code="TIMEOUT",
                message="No request received within 30 seconds",
                request_id=request_id,
            ))
            await websocket.close(code=1000)