Output:
Chunk ID: 6101e1
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 2878
Output:
#!/usr/bin/env python3
"""
Direct-runtime stress benchmark with detailed timing for optimized local TTS path.

This bypasses HTTP serving and calls `tts_runtime` directly so we can isolate
generation + decode bottlenecks under concurrency.
"""

from __future__ import annotations

import argparse
import asyncio
import json
import os
import statistics
import subprocess
import sys
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional

# Path setup (same pattern as local_server.py)
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
EXTERNAL_DIR = os.path.join(REPO_ROOT, "external")
for path in [os.path.join(EXTERNAL_DIR, "sparktts"), os.path.join(EXTERNAL_DIR, "AP-BWE"), REPO_ROOT]:
    if path not in sys.path:
        sys.path.insert(0, path)

# Keep local auth disabled to match local_server behavior
os.environ["AUTH_BYPASS_MODE"] = "true"

TEST_TEXTS = {
    "short": "Hello, this is a quick benchmark test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This benchmark sentence checks token generation speed, decode latency, "
        "and concurrency behavior for Spark TTS."
    ),
    "long": (
        "In the center of a crowded city, a quiet storyteller documented voices from dawn to dusk. "
        "Every pause, every laugh, every whisper revealed a different cadence. "
        "She replayed each recording at night, aligning tone and timing until each phrase felt natural."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    def __init__(self, interval_seconds: float = 0.5):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self) -> None:
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self) -> None:
        self._running = False
        if self._thread:
            self._thread.join(timeout=3)

    def _poll_loop(self) -> None:
        while self._running:
            try:
                res = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True,
                    text=True,
                    timeout=5,
                )
                if res.returncode == 0 and res.stdout:
                    parts = [p.strip() for p in res.stdout.strip().split(",")]
                    if len(parts) >= 4:
                        self.snapshots.append(
                            GPUSnapshot(
                                timestamp=time.time(),
                                memory_used_mb=float(parts[0]),
                                memory_total_mb=float(parts[1]),
                                gpu_utilization_pct=float(parts[2]),
                                temperature_c=float(parts[3]),
                            )
                        )
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {"samples": 0, "error": "no GPU samples captured"}
        mem = [s.memory_used_mb for s in self.snapshots]
        util = [s.gpu_utilization_pct for s in self.snapshots]
        temp = [s.temperature_c for s in self.snapshots]
        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem),
                "avg": statistics.mean(mem),
                "max": max(mem),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(util),
                "avg": statistics.mean(util),
                "max": max(util),
            },
            "temperature_c": {
                "min": min(temp),
                "avg": statistics.mean(temp),
                "max": max(temp),
            },
        }

@dataclass
class RequestResult:
    success: bool
    latency_ms: float
    audio_bytes: int
    audio_seconds: float
    speaker: str
    text_length: int
    error: Optional[str] = None
    timing: Dict[str, Any] = field(default_factory=dict)

@dataclass
class LevelResult:
    concurrency: int
    total_requests: int
    results: List[RequestResult] = field(default_factory=list)
    wall_time_s: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0.0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_s if self.wall_time_s > 0 else 0.0

    def p(self, values: List[float], pct: int) -> float:
        if not values:
            return 0.0
        s = sorted(values)
        idx = int((len(s) - 1) * (pct / 100))
        return s[idx]

    def timing_stats(self, key: str) -> Dict[str, float]:
        vals: List[float] = []
        for r in self.successes:
            v = r.timing.get(key)
            if isinstance(v, (int, float)):
                vals.append(float(v))
        if not vals:
            return {}
        return {
            "avg": statistics.mean(vals),
            "p50": self.p(vals, 50),
            "p95": self.p(vals, 95),
            "max": max(vals),
        }

    def latency_stats(self) -> Dict[str, float]:
        vals = [r.latency_ms for r in self.successes]
        if not vals:
            return {}
        return {
            "avg": statistics.mean(vals),
            "p50": self.p(vals, 50),
            "p95": self.p(vals, 95),
            "max": max(vals),
        }

def init_runtime(model_path: str, gpu_mem: float) -> None:
    from veena3modal.local_server import resolve_model_paths
    from veena3modal.services.tts_runtime import initialize_runtime, is_initialized

    llm_path, bicodec_path = resolve_model_paths(model_path)
    if not is_initialized():
        initialize_runtime(
            model_path=llm_path,
            bicodec_path=bicodec_path,
            device="cuda",
            gpu_memory_utilization=gpu_mem,
        )

async def run_one_request(
    text: str,
    speaker: str,
    max_tokens: int,
    timeout_s: float,
    chunking: bool,
) -> RequestResult:
    from veena3modal.services import tts_runtime

    t0 = time.perf_counter()
    try:
        if chunking:
            coro = tts_runtime.generate_speech_chunked(
                text=text,
                speaker=speaker,
                max_tokens=max_tokens,
            )
        else:
            coro = tts_runtime.generate_speech(
                text=text,
                speaker=speaker,
                max_tokens=max_tokens,
            )
        audio_bytes, timing = await asyncio.wait_for(coro, timeout=timeout_s)
        elapsed_ms = (time.perf_counter() - t0) * 1000
        if not audio_bytes:
            return RequestResult(
                success=False,
                latency_ms=elapsed_ms,
                audio_bytes=0,
                audio_seconds=0.0,
                speaker=speaker,
                text_length=len(text),
                error="no_audio",
                timing=timing or {},
            )
        sample_rate = int((timing or {}).get("output_sample_rate", 16000))
        audio_seconds = max(0.0, (len(audio_bytes) - 44) / (sample_rate * 2))
        return RequestResult(
            success=True,
            latency_ms=elapsed_ms,
            audio_bytes=len(audio_bytes),
            audio_seconds=audio_seconds,
            speaker=speaker,
            text_length=len(text),
            timing=timing or {},
        )
    except asyncio.TimeoutError:
        return RequestResult(
            success=False,
            latency_ms=(time.perf_counter() - t0) * 1000,
            audio_bytes=0,
            audio_seconds=0.0,
            speaker=speaker,
            text_length=len(text),
            error="timeout",
        )
    except Exception as exc:
        return RequestResult(
            success=False,
            latency_ms=(time.perf_counter() - t0) * 1000,
            audio_bytes=0,
            audio_seconds=0.0,
            speaker=speaker,
            text_length=len(text),
            error=str(exc)[:180],
        )

async def run_level(
    concurrency: int,
    total_requests: int,
    text: str,
    max_tokens: int,
    timeout_s: float,
    chunking: bool,
    gpu_monitor: GPUMonitor,
) -> LevelResult:
    level = LevelResult(concurrency=concurrency, total_requests=total_requests)
    semaphore = asyncio.Semaphore(concurrency)

    async def wrapped(i: int) -> RequestResult:
        async with semaphore:
            speaker = SPEAKERS[i % len(SPEAKERS)]
            return await run_one_request(
                text=text,
                speaker=speaker,
                max_tokens=max_tokens,
                timeout_s=timeout_s,
                chunking=chunking,
            )

    gpu_monitor.snapshots.clear()
    t0 = time.perf_counter()
    level.results = await asyncio.gather(*[wrapped(i) for i in range(total_requests)])
    level.wall_time_s = time.perf_counter() - t0
    level.gpu_summary = gpu_monitor.summary()
    return level

def print_level(level: LevelResult) -> None:
    print(f"\n=== concurrency={level.concurrency} requests={level.total_requests} ===")
    print(f"success={len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"wall={level.wall_time_s:.2f}s throughput={level.throughput_rps:.2f} req/s")
    lat = level.latency_stats()
    if lat:
        print(f"latency_ms avg={lat['avg']:.0f} p50={lat['p50']:.0f} p95={lat['p95']:.0f} max={lat['max']:.0f}")

    keys = [
        "generation_ms",
        "llm_batch_wall_ms_total",
        "llm_batch_gpu_ms_total",
        "llm_decode_wall_ms_total",
        "llm_decode_gpu_ms_total",
        "llm_parse_ms",
        "llm_time_per_token_ms",
        "bicodec_decode_wall_ms",
        "bicodec_decode_gpu_ms",
    ]
    for key in keys:
        st = level.timing_stats(key)
        if st:
            print(f"{key}: avg={st['avg']:.2f} p50={st['p50']:.2f} p95={st['p95']:.2f} max={st['max']:.2f}")

    if "memory_mb" in level.gpu_summary:
        g = level.gpu_summary
        print(
            "gpu: util_avg={:.0f}% util_max={:.0f}% mem_avg={:.0f}MB mem_max={:.0f}MB".format(
                g["gpu_util_pct"]["avg"],
                g["gpu_util_pct"]["max"],
                g["memory_mb"]["avg"],
                g["memory_mb"]["max"],
            )
        )

    errors: Dict[str, int] = {}
    for r in level.failures:
        k = r.error or "failed"
        errors[k] = errors.get(k, 0) + 1
    for k, v in sorted(errors.items(), key=lambda kv: -kv[1]):
        print(f"error[{v}] {k}")


Output:
Chunk ID: fb4101
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 3557
Output:
#!/usr/bin/env python3
"""Detailed local stress benchmark for optimized TTS path.

This script is focused on collecting per-request GPU/LLM timing for local tuning:
- Request wall-clock latency and server reported TTFB
- GPU timing breakdown (LLM batch wall / GPU wall, parse, decode)
- Time per token and per batch
- Concurrency visibility from in-flight counters
- End-to-end throughput and RTF with GPU utilization snapshots

Usage:
    python scripts/stress_test_optimized_local.py
    python scripts/stress_test_optimized_local.py --levels 1,5,10,20 --concurrency-scaling both
    python scripts/stress_test_optimized_local.py --text-category long --output benchmarks.json
"""

import argparse
import asyncio
import json
import statistics
import subprocess
import sys
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple

import httpx

DEFAULT_URL = "http://localhost:8000"
GENERATE_PATH = "/v1/tts/generate"

TEST_TEXTS = {
    "short": "Hello, this is a quick test.",
    "medium": (
        "The quick brown fox jumps over the lazy dog. "
        "This is a baseline quality sentence used for benchmarking local TTS latency, "
        "TTFB, and tokenization behavior under load."
    ),
    "long": (
        "In the heart of a bustling city, where towering skyscrapers cast shadows over crowded streets, "
        "there lived a quiet linguist who believed every sentence carried a rhythm. "
        "She recorded conversations at dawn and dusk, trying to capture how meaning shifted with breath. "
        "By evening she would review them, matching sound to emotion, and every night she learned a new way to speak."
    ),
}

SPEAKERS = ["lipakshi", "vardan", "reet", "Nandini", "krishna", "anika"]

@dataclass
class GPUSnapshot:
    timestamp: float
    memory_used_mb: float
    memory_total_mb: float
    gpu_utilization_pct: float
    temperature_c: float

class GPUMonitor:
    """Background polling for quick GPU health snapshots."""

    def __init__(self, interval_seconds: float = 1.0):
        self.interval = interval_seconds
        self.snapshots: List[GPUSnapshot] = []
        self._running = False
        self._thread: Optional[threading.Thread] = None

    def start(self):
        self._running = True
        self._thread = threading.Thread(target=self._poll_loop, daemon=True)
        self._thread.start()

    def stop(self):
        self._running = False
        if self._thread:
            self._thread.join(timeout=3)

    def _poll_loop(self):
        while self._running:
            try:
                result = subprocess.run(
                    [
                        "nvidia-smi",
                        "--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu",
                        "--format=csv,noheader,nounits",
                    ],
                    capture_output=True,
                    text=True,
                    timeout=5,
                )
                if result.returncode == 0 and result.stdout:
                    parts = [p.strip() for p in result.stdout.strip().split(",")]
                    if len(parts) >= 4:
                        self.snapshots.append(
                            GPUSnapshot(
                                timestamp=time.time(),
                                memory_used_mb=float(parts[0]),
                                memory_total_mb=float(parts[1]),
                                gpu_utilization_pct=float(parts[2]),
                                temperature_c=float(parts[3]),
                            )
                        )
            except Exception:
                pass
            time.sleep(self.interval)

    def summary(self) -> Dict[str, Any]:
        if not self.snapshots:
            return {
                "samples": 0,
                "error": "no GPU samples captured",
            }

        mem_used = [s.memory_used_mb for s in self.snapshots]
        gpu_util = [s.gpu_utilization_pct for s in self.snapshots]
        temp = [s.temperature_c for s in self.snapshots]

        return {
            "samples": len(self.snapshots),
            "memory_mb": {
                "min": min(mem_used),
                "max": max(mem_used),
                "avg": statistics.mean(mem_used),
                "total": self.snapshots[0].memory_total_mb,
            },
            "gpu_util_pct": {
                "min": min(gpu_util),
                "max": max(gpu_util),
                "avg": statistics.mean(gpu_util),
            },
            "temperature_c": {
                "min": min(temp),
                "max": max(temp),
                "avg": statistics.mean(temp),
            },
        }

@dataclass
class RequestResult:
    success: bool
    status_code: int
    latency_ms: float
    ttfb_ms: float = 0.0
    server_ttfb_ms: float = 0.0
    audio_bytes: int = 0
    audio_seconds: float = 0.0
    request_id: str = ""
    error: Optional[str] = None
    speaker: str = ""
    text_length: int = 0
    stream: bool = False
    timing: Dict[str, Any] = field(default_factory=dict)
    request_inflight: Optional[int] = None

@dataclass
class LevelResult:
    level_name: str
    concurrency: int
    total_requests: int
    mode: str
    text_category: str
    chunking: bool
    results: List[RequestResult] = field(default_factory=list)
    wall_time_seconds: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0.0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_seconds if self.wall_time_seconds > 0 else 0.0

    @property
    def audio_duration_total(self) -> float:
        return sum(r.audio_seconds for r in self.successes)

    @property
    def effective_rtf(self) -> float:
        total_audio = self.audio_duration_total
        return self.wall_time_seconds / total_audio if total_audio > 0 else 0.0

    def _percentile(self, values: List[float], pct: float) -> float:
        if not values:
            return 0.0
        sorted_vals = sorted(values)
        idx = int(len(sorted_vals) * pct / 100)
        return sorted_vals[min(idx, len(sorted_vals) - 1)]

    @property
    def latency_stats(self) -> Dict[str, float]:
        lat = [r.latency_ms for r in self.successes]
        if not lat:
            return {}
        return {
            "min": min(lat),
            "avg": statistics.mean(lat),
            "p50": self._percentile(lat, 50),
            "p95": self._percentile(lat, 95),
            "p99": self._percentile(lat, 99),
            "max": max(lat),
        }

    @property
    def ttfb_stats(self) -> Dict[str, float]:
        vals = [r.server_ttfb_ms for r in self.successes if r.server_ttfb_ms >= 0]
        if not vals:
            return {}
        return {
            "min": min(vals),
            "avg": statistics.mean(vals),
            "p50": self._percentile(vals, 50),
            "p95": self._percentile(vals, 95),
            "max": max(vals),
        }

    def timing_stats(self, key: str) -> Dict[str, float]:
        values: List[float] = []
        for r in self.successes:
            v = r.timing.get(key)
            if isinstance(v, (int, float)):
                values.append(float(v))
        if not values:
            return {}
        return {
            "min": min(values),
            "avg": statistics.mean(values),
            "p50": self._percentile(values, 50),
            "p95": self._percentile(values, 95),
            "p99": self._percentile(values, 99),
            "max": max(values),
        }

    def timing_rate(self, numerator_key: str, denominator_key: str) -> Tuple[float, int]:
        num = 0.0
        den = 0
        for r in self.successes:
            n = r.timing.get(numerator_key)
            d = r.timing.get(denominator_key)
            if isinstance(n, (int, float)) and isinstance(d, (int, float)) and d > 0:
                num += float(n)
                den += int(d)
        if den == 0:
            return 0.0, 0
        return num / den, den

    def observed_inflight_max(self) -> Optional[int]:
        values = [r.request_inflight for r in self.successes if isinstance(r.request_inflight, int)]
        if not values:
            return None
        return max(values)

    def error_summary(self) -> Dict[str, int]:
        errors: Dict[str, int] = {}
        for r in self.failures:
            key = r.error or f"HTTP {r.status_code}"
            errors[key[:100]] = errors.get(key[:100], 0) + 1
        return errors

def _coerce_number(raw: str) -> Optional[float]:
    if raw is None:
        return None
    v = str(raw).strip()
    if v == "":
        return None
    try:
        if "." in v:
            return float(v)
        return float(int(v))
    except ValueError:
        try:
            return float(v)
        except ValueError:
            return None

def _normalize_time_ms(raw: Optional[str]) -> float:
    value = _coerce_number(raw)
    if value is None:
        return 0.0
    return float(value)

def _parse_perf_headers(headers: Any) -> Dict[str, Any]:
    """Parse server timing headers into numeric metric fields."""
    metrics: Dict[str, Any] = {}

    # Explicit compact payload.
    detail_payload = headers.get("x-perf-details") if hasattr(headers, "get") else None
    if detail_payload:
        try:
            parsed = json.loads(detail_payload)
            if isinstance(parsed, dict):
                metrics.update(parsed)
        except Exception:
            pass

    header_map = {
        "x-llm-token-total": "llm_token_total",
        "x-llm-batch-count": "llm_batch_count",
        "x-llm-batch-wall-ms": "llm_batch_wall_ms_total",
        "x-llm-batch-gpu-ms": "llm_batch_gpu_ms_total",
        "x-llm-batch-wall-ms-min": "llm_batch_wall_ms_min",
        "x-llm-batch-wall-ms-max": "llm_batch_wall_ms_max",
        "x-llm-batch-wall-ms-p50": "llm_batch_wall_ms_p50",
        "x-llm-batch-gpu-ms-min": "llm_batch_gpu_ms_min",
        "x-llm-batch-gpu-ms-max": "llm_batch_gpu_ms_max",
        "x-llm-batch-gpu-ms-p50": "llm_batch_gpu_ms_p50",
        "x-llm-decode-wall-ms": "llm_decode_wall_ms_total",
        "x-llm-decode-gpu-ms": "llm_decode_gpu_ms_total",
        "x-llm-decode-wall-ms-min": "llm_decode_wall_ms_min",
        "x-llm-decode-wall-ms-max": "llm_decode_wall_ms_max",
        "x-llm-decode-wall-ms-p50": "llm_decode_wall_ms_p50",
        "x-llm-decode-gpu-ms-min": "llm_decode_gpu_ms_min",
        "x-llm-decode-gpu-ms-max": "llm_decode_gpu_ms_max",
        "x-llm-decode-gpu-ms-p50": "llm_decode_gpu_ms_p50",
        "x-llm-decode-calls": "decode_calls",
        "x-llm-decode-cpu-ms": "llm_decode_cpu_ms",
        "x-llm-parse-ms": "llm_parse_ms",
        "x-llm-parse-avg-ms": "llm_parse_ms_avg",
        "x-llm-tokens-per-batch-min": "tokens_per_batch_min",
        "x-llm-tokens-per-batch-max": "tokens_per_batch_max",
        "x-llm-tokens-per-batch-p50": "tokens_per_batch_p50",
        "x-generation-ms": "generation_ms",
        "x-request-inflight": "request_inflight",
        "x-llm-text-chunked": "text_chunked",
        "x-llm-chunks-processed": "chunks_processed",
        "x-llm-time-per-token-ms": "llm_time_per_token_ms",
        "x-llm-time-per-batch-wall-ms": "llm_time_per_batch_wall_ms",
        "x-llm-time-per-batch-gpu-ms": "llm_time_per_batch_gpu_ms",
        "x-llm-parse-avg-ms": "llm_parse_ms_avg",
        "x-bicodec-decode-wall-ms": "bicodec_decode_wall_ms",
        "x-bicodec-decode-gpu-ms": "bicodec_decode_gpu_ms",
        "x-bicodec-decode-cpu-ms": "bicodec_decode_cpu_ms",
    }

    if hasattr(headers, "items"):
        for raw_key, raw_val in headers.items():
            lk = str(raw_key).lower()
            target_key = header_map.get(lk)
            if target_key is None:
                continue
            v = _coerce_number(raw_val)
            if v is not None:
                if target_key in {
                    "text_chunked",
                }:
                    metrics[target_key] = bool(v)
                else:
                    metrics[target_key] = int(v) if float(v).is_integer() else float(v)

    if "llm_token_total" in metrics and "generation_ms" in metrics and metrics["llm_token_total"]:
        try:
            metrics["llm_time_per_token_ms"] = float(metrics["generation_ms"]) / float(metrics["llm_token_total"])
        except Exception:
            pass

    if "llm_batch_count" in metrics and "generation_ms" in metrics and metrics["llm_batch_count"]:
        try:
            metrics["llm_time_per_batch_wall_ms"] = float(metrics["generation_ms"]) / float(metrics["llm_batch_count"])
        except Exception:
            pass

    if "llm_batch_count" in metrics and "llm_batch_gpu_ms_total" in metrics and metrics["llm_batch_count"]:
        try:
            metrics["llm_time_per_batch_gpu_ms"] = float(metrics["llm_batch_gpu_ms_total"]) / float(metrics["llm_batch_count"])
        except Exception:
            pass

    return metrics

async def make_request(
    client: httpx.AsyncClient,
    base_url: str,
    text: str,
    speaker: str,
    chunking: bool,
    stream: bool = False,
    timeout: float = 120.0,
) -> RequestResult:
    url = f"{base_url}{GENERATE_PATH}"
    payload = {
        "text": text,
        "speaker": speaker,
        "stream": stream,
        "chunking": chunking,
        "format": "wav",
    }

    start = time.time()
    ttfb_time = 0.0

    try:
        if stream:
            async with client.stream(
                "POST",
                url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            ) as response:
                first = True
                total_bytes = 0
                async for chunk in response.aiter_bytes():


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Inspecting token limit handling**


sed -n '360,820p' scripts/stress_test_runtime_detailed.py


sed -n '420,980p' scripts/stress_test_optimized_local.py


Output:
Chunk ID: 628182
Wall time: 0.0517 seconds
Process exited with code 0
Original token count: 1185
Output:

def save_results(path: str, levels: List[LevelResult], args: argparse.Namespace) -> None:
    payload = {
        "config": {
            "levels": args.levels,
            "text": args.text,
            "max_tokens": args.max_tokens,
            "timeout_s": args.timeout,
            "chunking": args.chunking,
            "requests_multiplier": args.requests_multiplier,
            "timestamp": int(time.time()),
        },
        "levels": [],
    }
    for lvl in levels:
        payload["levels"].append(
            {
                "concurrency": lvl.concurrency,
                "total_requests": lvl.total_requests,
                "success_rate": lvl.success_rate,
                "throughput_rps": lvl.throughput_rps,
                "wall_time_s": lvl.wall_time_s,
                "latency_ms": lvl.latency_stats(),
                "gpu": lvl.gpu_summary,
                "timing": {
                    "generation_ms": lvl.timing_stats("generation_ms"),
                    "llm_batch_wall_ms_total": lvl.timing_stats("llm_batch_wall_ms_total"),
                    "llm_batch_gpu_ms_total": lvl.timing_stats("llm_batch_gpu_ms_total"),
                    "llm_decode_wall_ms_total": lvl.timing_stats("llm_decode_wall_ms_total"),
                    "llm_decode_gpu_ms_total": lvl.timing_stats("llm_decode_gpu_ms_total"),
                    "llm_parse_ms": lvl.timing_stats("llm_parse_ms"),
                    "llm_time_per_token_ms": lvl.timing_stats("llm_time_per_token_ms"),
                    "bicodec_decode_wall_ms": lvl.timing_stats("bicodec_decode_wall_ms"),
                    "bicodec_decode_gpu_ms": lvl.timing_stats("bicodec_decode_gpu_ms"),
                },
                "requests": [
                    {
                        "success": r.success,
                        "latency_ms": r.latency_ms,
                        "audio_bytes": r.audio_bytes,
                        "audio_seconds": r.audio_seconds,
                        "speaker": r.speaker,
                        "text_length": r.text_length,
                        "error": r.error,
                        "timing": r.timing,
                    }
                    for r in lvl.results
                ],
            }
        )
    with open(path, "w", encoding="utf-8") as fd:
        json.dump(payload, fd, indent=2)

async def main_async(args: argparse.Namespace) -> None:
    text = TEST_TEXTS[args.text]
    levels = [int(x.strip()) for x in args.levels.split(",") if x.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    # warmup
    await run_one_request(
        text=text,
        speaker=SPEAKERS[0],
        max_tokens=args.max_tokens,
        timeout_s=args.timeout,
        chunking=args.chunking,
    )

    results: List[LevelResult] = []
    for idx, c in enumerate(levels):
        n = max(args.min_requests, c * args.requests_multiplier)
        level = await run_level(
            concurrency=c,
            total_requests=n,
            text=text,
            max_tokens=args.max_tokens,
            timeout_s=args.timeout,
            chunking=args.chunking,
            gpu_monitor=gpu_monitor,
        )
        print_level(level)
        results.append(level)
        if idx < len(levels) - 1:
            await asyncio.sleep(2)

    gpu_monitor.stop()
    save_results(args.output, results, args)
    print(f"\nresults saved: {args.output}")

def main() -> None:
    parser = argparse.ArgumentParser(description="Direct runtime stress benchmark (detailed timings)")
    parser.add_argument("--levels", default="1,2,4,8", help="Comma-separated concurrency levels")
    parser.add_argument("--text", choices=["short", "medium", "long"], default="short")
    parser.add_argument("--max-tokens", type=int, default=128, help="max_tokens sent to generation")
    parser.add_argument("--timeout", type=float, default=120.0, help="Per-request timeout seconds")
    parser.add_argument("--requests-multiplier", type=int, default=4, help="Requests per level ~= concurrency * multiplier")
    parser.add_argument("--min-requests", type=int, default=8)
    parser.add_argument("--chunking", action="store_true", default=False, help="Use generate_speech_chunked")
    parser.add_argument("--gpu-memory", type=float, default=0.25)
    parser.add_argument("--model-path", type=str, default="")
    parser.add_argument("--output", default="stress_runtime_detailed.json")
    args = parser.parse_args()

    from veena3modal.local_server import DEFAULT_LOCAL_MODEL_DIR
    model_path = args.model_path or DEFAULT_LOCAL_MODEL_DIR
    init_runtime(model_path=model_path, gpu_mem=args.gpu_memory)

    asyncio.run(main_async(args))

if __name__ == "__main__":
    main()


Output:
Chunk ID: 11f107
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 4155
Output:
                async for chunk in response.aiter_bytes():
                    if first:
                        ttfb_time = (time.time() - start) * 1000
                        first = False
                    total_bytes += len(chunk)

                latency = (time.time() - start) * 1000
                timing = _parse_perf_headers(response.headers)

                if response.status_code == 200:
                    pcm_bytes = max(0, total_bytes - 44)
                    audio_seconds = pcm_bytes / (16000 * 2)
                    return RequestResult(
                        success=True,
                        status_code=200,
                        latency_ms=latency,
                        ttfb_ms=ttfb_time,
                        audio_bytes=total_bytes,
                        audio_seconds=audio_seconds,
                        request_id=response.headers.get("x-request-id", ""),
                        speaker=speaker,
                        text_length=len(text),
                        stream=True,
                        timing=timing,
                        request_inflight=timing.get("request_inflight"),
                        server_ttfb_ms=_normalize_time_ms(response.headers.get("x-ttfb-ms")),
                    )

                return RequestResult(
                    success=False,
                    status_code=response.status_code,
                    latency_ms=latency,
                    ttfb_ms=ttfb_time,
                    error=f"HTTP {response.status_code}",
                    speaker=speaker,
                    text_length=len(text),
                    stream=True,
                    timing=timing,
                    request_id=response.headers.get("x-request-id", ""),
                    request_inflight=timing.get("request_inflight"),
                )
        else:
            response = await client.post(
                url,
                json=payload,
                headers={"Content-Type": "application/json"},
                timeout=timeout,
            )
            latency = (time.time() - start) * 1000
            timing = _parse_perf_headers(response.headers)

            server_ttfb = _normalize_time_ms(response.headers.get("x-ttfb-ms"))
            audio_seconds = 0.0
            audio_sec_header = response.headers.get("x-audio-seconds")
            if audio_sec_header:
                parsed = _coerce_number(audio_sec_header)
                if parsed is not None:
                    audio_seconds = parsed
            elif response.status_code == 200:
                pcm_bytes = max(0, len(response.content) - 44)
                audio_seconds = pcm_bytes / (16000 * 2)

            if response.status_code == 200:
                return RequestResult(
                    success=True,
                    status_code=200,
                    latency_ms=latency,
                    ttfb_ms=latency,
                    server_ttfb_ms=server_ttfb,
                    audio_bytes=len(response.content),
                    audio_seconds=audio_seconds,
                    request_id=response.headers.get("x-request-id", ""),
                    speaker=speaker,
                    text_length=len(text),
                    stream=False,
                    timing=timing,
                    request_inflight=timing.get("request_inflight"),
                )

            return RequestResult(
                success=False,
                status_code=response.status_code,
                latency_ms=latency,
                ttfb_ms=latency,
                server_ttfb_ms=server_ttfb,
                audio_bytes=len(response.content),
                error=(response.text[:200] if response.text else f"HTTP {response.status_code}"),
                speaker=speaker,
                text_length=len(text),
                stream=False,
                timing=timing,
                request_id=response.headers.get("x-request-id", ""),
                request_inflight=timing.get("request_inflight"),
            )

    except httpx.TimeoutException:
        return RequestResult(
            success=False,
            status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error="TIMEOUT",
            speaker=speaker,
            text_length=len(text),
            stream=stream,
        )
    except Exception as e:
        return RequestResult(
            success=False,
            status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error=str(e)[:200],
            speaker=speaker,
            text_length=len(text),
            stream=stream,
        )

async def run_level(
    base_url: str,
    num_requests: int,
    concurrency: int,
    text_category: str,
    chunking: bool,
    stream: bool,
    gpu_monitor: Optional[GPUMonitor] = None,
) -> LevelResult:
    mode = "streaming" if stream else "non-streaming"
    level = LevelResult(
        level_name=f"{concurrency}c-{text_category}-{mode}",
        concurrency=concurrency,
        total_requests=num_requests,
        mode=mode,
        text_category=text_category,
        chunking=chunking,
    )

    semaphore = asyncio.Semaphore(concurrency)

    if text_category == "mixed":
        texts = list(TEST_TEXTS.values())
    else:
        texts = [TEST_TEXTS.get(text_category, TEST_TEXTS["short"])]

    async def limited_request(i: int, client: httpx.AsyncClient) -> RequestResult:
        async with semaphore:
            text = texts[i % len(texts)]
            speaker = SPEAKERS[i % len(SPEAKERS)]
            return await make_request(
                client=client,
                base_url=base_url,
                text=text,
                speaker=speaker,
                chunking=chunking,
                stream=stream,
            )

    if gpu_monitor:
        gpu_monitor.snapshots.clear()

    start_time = time.time()
    limits = httpx.Limits(
        max_connections=concurrency + 10,
        max_keepalive_connections=concurrency,
    )
    async with httpx.AsyncClient(limits=limits) as client:
        tasks = [limited_request(i, client) for i in range(num_requests)]
        responses = await asyncio.gather(*tasks, return_exceptions=True)

    level.wall_time_seconds = time.time() - start_time

    for item in responses:
        if isinstance(item, RequestResult):
            level.results.append(item)
        else:
            level.results.append(
                RequestResult(
                    success=False,
                    status_code=0,
                    latency_ms=0,
                    error=str(item)[:200],
                    stream=stream,
                )
            )

    if gpu_monitor:
        level.gpu_summary = gpu_monitor.summary()

    return level

def print_level_report(level: LevelResult):
    print(f"\n{'=' * 72}")
    print(f"  {level.level_name.upper()}")
    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}, chunking={level.chunking}")
    print(f"{'=' * 72}")

    print(f"  Success:     {len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"  Wall time:   {level.wall_time_seconds:.2f}s")
    print(f"  Throughput:  {level.throughput_rps:.2f} req/s")
    print(f"  Audio total: {level.audio_duration_total:.2f}s")
    print(f"  Eff. RTF:    {level.effective_rtf:.3f}")

    if level.observed_inflight_max() is not None:
        print(f"  Peak inflight: {level.observed_inflight_max()}")

    latency = level.latency_stats
    if latency:
        print("\n  Latency (ms):")
        print(
            f"    min={latency['min']:.0f} avg={latency['avg']:.0f} "
            f"p50={latency['p50']:.0f} p95={latency['p95']:.0f} p99={latency['p99']:.0f} max={latency['max']:.0f}"
        )

    ttfb = level.ttfb_stats
    if ttfb and not level.mode.startswith("stream"):
        print("\n  Server TTFB (ms):")
        print(
            f"    min={ttfb['min']:.0f} avg={ttfb['avg']:.0f} "
            f"p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}"
        )

    if level.mode == "non-streaming":
        llm = level.timing_stats("generation_ms")
        if llm:
            print("\n  Generation (ms):")
            print(f"    min={llm['min']:.0f} avg={llm['avg']:.0f} p50={llm['p50']:.0f} p95={llm['p95']:.0f} max={llm['max']:.0f}")
        wall = level.timing_stats("llm_batch_wall_ms_total")
        if wall:
            print("\n  LLM batch wall (ms):")
            print(f"    min={wall['min']:.0f} avg={wall['avg']:.0f} p50={wall['p50']:.0f} p95={wall['p95']:.0f} max={wall['max']:.0f}")
        gpu = level.timing_stats("llm_batch_gpu_ms_total")
        if gpu:
            print("\n  LLM batch GPU (ms):")
            print(f"    min={gpu['min']:.0f} avg={gpu['avg']:.0f} p50={gpu['p50']:.0f} p95={gpu['p95']:.0f} max={gpu['max']:.0f}")
        parse = level.timing_stats("llm_parse_ms")
        if parse:
            print("\n  Parse (ms):")
            print(f"    avg={parse['avg']:.0f} p95={parse['p95']:.0f}")
        decode = level.timing_stats("llm_decode_wall_ms_total")
        if decode:
            print("\n  Decode (ms):")
            print(f"    avg={decode['avg']:.0f} p95={decode['p95']:.0f}")
        bdecode = level.timing_stats("bicodec_decode_wall_ms")
        if bdecode:
            print("\n  BiCodec decode (ms):")
            print(f"    avg={bdecode['avg']:.0f} p95={bdecode['p95']:.0f}")
        bdecode_gpu = level.timing_stats("bicodec_decode_gpu_ms")
        if bdecode_gpu:
            print("\n  BiCodec decode GPU (ms):")
            print(f"    avg={bdecode_gpu['avg']:.0f} p95={bdecode_gpu['p95']:.0f}")

        token_ms, token_count = level.timing_rate("generation_ms", "llm_token_total")
        if token_count:
            print(f"  Token timing: avg {token_ms * 1000:.3f} us/token")

    if level.gpu_summary and "memory_mb" in level.gpu_summary:
        gpu_summary = level.gpu_summary
        mem = gpu_summary["memory_mb"]
        util = gpu_summary["gpu_util_pct"]
        temp = gpu_summary["temperature_c"]
        print("\n  GPU:")
        print(f"    Memory: {mem['avg']:.0f}MB avg, {mem['max']:.0f}MB peak / {mem['total']:.0f}MB")
        print(f"    Util:   {util['avg']:.0f}% avg, {util['max']:.0f}% peak")
        print(f"    Temp:   {temp['avg']:.0f}C avg, {temp['max']:.0f}C peak")

    errors = level.error_summary()
    if errors:
        print("\n  Errors:")
        for key, count in sorted(errors.items(), key=lambda kv: -kv[1]):
            print(f"    [{count}x] {key}")

def print_summary_table(levels: List[LevelResult]):
    print(f"\n{'=' * 110}")
    print("  STRESS BENCHMARK SUMMARY")
    print(f"{'=' * 110}")
    header = f"{'Level':<26} {'OK%':>5} {'RPS':>7} {'p50':>7} {'p95':>7} {'GPU%':>5} {'GPUmem':>8} {'RTF':>6} {'Inflight':>8}"
    print(header)
    print("-" * 110)

    for level in levels:
        lat = level.latency_stats
        p50 = f"{lat.get('p50', 0):.0f}" if lat else "---"
        p95 = f"{lat.get('p95', 0):.0f}" if lat else "---"
        gpu_util = level.gpu_summary.get("gpu_util_pct", {}).get("avg", 0)
        gpu_mem = level.gpu_summary.get("memory_mb", {}).get("max", 0)
        inflight = level.observed_inflight_max() or 0
        print(
            f"{level.level_name:<26} "
            f"{level.success_rate:>4.0%} "
            f"{level.throughput_rps:>7.2f} "
            f"{p50:>7} "
            f"{p95:>7} "
            f"{gpu_util:>4.0f}% "
            f"{gpu_mem:>7.0f}MB "
            f"{level.effective_rtf:>6.3f} "
            f"{inflight:>8d}"
        )

    print("=" * 110)

def save_results(levels: List[LevelResult], path: str):
    payload = []
    for level in levels:
        payload.append({
            "level": level.level_name,
            "concurrency": level.concurrency,
            "mode": level.mode,
            "chunking": level.chunking,
            "text_category": level.text_category,
            "success_rate": level.success_rate,
            "throughput_rps": level.throughput_rps,
            "wall_time_s": level.wall_time_seconds,
            "latency_ms": level.latency_stats,
            "ttfb_ms": level.ttfb_stats,
            "effective_rtf": level.effective_rtf,
            "audio_total_seconds": level.audio_duration_total,
            "observed_inflight_max": level.observed_inflight_max(),
            "errors": level.error_summary(),
            "gpu": level.gpu_summary,
            "timing_averages": {
                "generation_ms": level.timing_stats("generation_ms"),
                "llm_batch_wall_ms_total": level.timing_stats("llm_batch_wall_ms_total"),
                "llm_batch_gpu_ms_total": level.timing_stats("llm_batch_gpu_ms_total"),
                "llm_parse_ms": level.timing_stats("llm_parse_ms"),
                "llm_decode_wall_ms_total": level.timing_stats("llm_decode_wall_ms_total"),
                "bicodec_decode_wall_ms": level.timing_stats("bicodec_decode_wall_ms"),
                "bicodec_decode_gpu_ms": level.timing_stats("bicodec_decode_gpu_ms"),
            },
            "requests": [
                {
                    "request_id": r.request_id,
                    "success": r.success,
                    "status_code": r.status_code,
                    "latency_ms": r.latency_ms,
                    "ttfb_ms": r.ttfb_ms,
                    "server_ttfb_ms": r.server_ttfb_ms,
                    "audio_bytes": r.audio_bytes,
                    "audio_seconds": r.audio_seconds,
                    "request_inflight": r.request_inflight,
                    "error": r.error,
                    "timing": r.timing,
                }
                for r in level.results
            ],
        })

    with open(path, "w") as fd:
        json.dump(payload, fd, indent=2)

async def run_stress_test(args):
    base_url = args.url

    try:
        async with httpx.AsyncClient() as client:
            response = await client.get(f"{base_url}/v1/tts/health", timeout=10)
            health = response.json()
            print(f"  Health: {health.get('status')}")
            print(f"  Version: {health.get('model_version')}")
            print(f"  GPU: {health.get('gpu_available')}")
    except Exception as e:
        print(f"ERROR: server not reachable at {base_url}: {e}")
        sys.exit(1)

    levels = [int(v) for v in args.levels.split(",") if v.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    mode_label = "streaming" if args.stream else "non-streaming"
    results: List[LevelResult] = []

    print(f"\nWarming up (1 request, non-streaming)...")
    warmup = await run_level(
        base_url=base_url,
        num_requests=1,
        concurrency=1,
        text_category=args.text,
        chunking=args.chunking,
        stream=False,
        gpu_monitor=gpu_monitor,
    )
    print(f"  warmup ok: {warmup.success_rate:.0%}")

    for conc in levels:
        num_requests = max(conc * 3, 10)
        num_requests = min(num_requests, 200)
        level = await run_level(
            base_url=base_url,
            num_requests=num_requests,
            concurrency=conc,
            text_category=args.text,
            chunking=args.chunking,
            stream=args.stream,
            gpu_monitor=gpu_monitor,
        )
        print_level_report(level)
        results.append(level)
        if conc != levels[-1]:
            print("  cooldown 3s...")
            await asyncio.sleep(3)

    gpu_monitor.stop()
    print_summary_table(results)

    if args.output:
        save_results(results, args.output)
        print(f"\nResults written to {args.output}")

def main():
    parser = argparse.ArgumentParser(description="Detailed local optimized-path TTS benchmark")
    parser.add_argument("--url", default=DEFAULT_URL, help=f"Server URL (default: {DEFAULT_URL})")
    parser.add_argument("--levels", default="1,5,10,20,50", help="Comma-separated concurrency levels")
    parser.add_argument("--text", default="short", choices=["short", "medium", "long", "mixed"], help="Input text category")
    parser.add_argument("--chunking", action="store_true", help="Enable server-side chunking", default=True)
    parser.add_argument("--no-chunking", action="store_false", dest="chunking", help="Disable chunking")
    parser.add_argument("--stream", action="store_true", help="Use streaming endpoint")
    parser.add_argument("--output", default="stress_test_optimized_local.json", help="Output JSON path")

    args = parser.parse_args()
    asyncio.run(run_stress_test(args))

if __name__ == "__main__":
    main()


Summary: auto


**Inspecting local server defaults**


sed -n '1,360p' veena3modal/local_server.py


Output:
Chunk ID: 68aa09
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 3200
Output:
"""
Local entry point for Veena3 TTS - runs the same FastAPI app without Modal.

Replicates what Modal's @modal.enter + @modal.asgi_app does:
1. Sets up PYTHONPATH for vendored deps (sparktts, AP-BWE)
2. Downloads model weights from HuggingFace if not present
3. Initializes TTS runtime (vLLM engine, BiCodec decoder, pipelines)
4. Serves the FastAPI app via uvicorn

Usage:
    # Basic (auto-downloads model, uses defaults)
    python -m veena3modal.local_server

    # Custom model path (skip download)
    python -m veena3modal.local_server --model-path /path/to/spark_tts_4speaker

    # Custom GPU memory + port
    python -m veena3modal.local_server --gpu-memory 0.5 --port 8080

    # With super-resolution (48kHz output)
    python -m veena3modal.local_server --enable-sr --sr-path /path/to/ap_bwe/16kto48k

    # CPU-only (no GPU, for testing only - very slow)
    python -m veena3modal.local_server --device cpu

Environment Variables (override defaults):
    MODEL_PATH              - Path to Spark TTS model directory
    HF_TOKEN                - HuggingFace token for private models
    AP_BWE_CHECKPOINT_DIR   - Path to super-resolution checkpoints
    AUTH_BYPASS_MODE        - "true" to disable auth (default for local)
    GPU_MEMORY_UTILIZATION  - vLLM GPU memory fraction (default: 0.25)
"""

from __future__ import annotations

import argparse
import logging
import os
import sys
import time

# === Path Setup (MUST happen before any veena3modal imports) ===
# Vendored deps: sparktts and AP-BWE live in external/ relative to repo root
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
EXTERNAL_DIR = os.path.join(REPO_ROOT, "external")

# sparktts package path: external/sparktts (contains sparktts/ subpackage)
SPARKTTS_PATH = os.path.join(EXTERNAL_DIR, "sparktts")
# AP-BWE path: external/AP-BWE
AP_BWE_PATH = os.path.join(EXTERNAL_DIR, "AP-BWE")

for path in [SPARKTTS_PATH, AP_BWE_PATH, REPO_ROOT]:
    if path not in sys.path:
        sys.path.insert(0, path)

# Also set PYTHONPATH for subprocesses (e.g., ffmpeg wrappers)
existing_pythonpath = os.environ.get("PYTHONPATH", "")
new_paths = f"{SPARKTTS_PATH}:{AP_BWE_PATH}:{REPO_ROOT}"
if existing_pythonpath:
    os.environ["PYTHONPATH"] = f"{new_paths}:{existing_pythonpath}"
else:
    os.environ["PYTHONPATH"] = new_paths

logger = logging.getLogger("veena3.local")

# Default local model directory (relative to repo)
DEFAULT_LOCAL_MODEL_DIR = os.path.join(REPO_ROOT, "models", "spark_tts_4speaker")

# Model sources (tried in order):
# 1. Local path (--model-path or MODEL_PATH env)
# 2. HuggingFace download (private repo: BayAreaBoys/spark_tts_4speaker)
# 3. Base Spark TTS model (public: SparkAudio/Spark-TTS-0.5B)
PRIVATE_HF_REPO = "BayAreaBoys/spark_tts_4speaker"
PUBLIC_FALLBACK_REPO = "SparkAudio/Spark-TTS-0.5B"

def download_model(
    hf_repo: str = PRIVATE_HF_REPO,
    local_dir: str = DEFAULT_LOCAL_MODEL_DIR,
    hf_token: str | None = None,
) -> str:
    """
    Download Spark TTS model from HuggingFace if not already present.

    Strategy:
    1. Try private repo (BayAreaBoys/spark_tts_4speaker) with HF token
    2. Fall back to public base model (SparkAudio/Spark-TTS-0.5B)
    3. Provide instructions for Modal volume download if both fail

    Returns the resolved local path.
    """
    # Check for any valid marker file (config.json at root, config.yaml, or LLM/config.json)
    marker_files = ["config.json", "config.yaml", "LLM/config.json"]
    if any(os.path.exists(os.path.join(local_dir, m)) for m in marker_files):
        logger.info(f"Model already present at {local_dir}")
        return local_dir

    try:
        from huggingface_hub import snapshot_download
    except ImportError:
        logger.error(
            "huggingface_hub not installed. Install with: pip install huggingface-hub\n"
            "Or download model manually and pass --model-path"
        )
        sys.exit(1)

    os.makedirs(local_dir, exist_ok=True)

    # Try private repo first
    logger.info(f"Attempting download from {hf_repo}...")
    try:
        downloaded_path = snapshot_download(
            repo_id=hf_repo,
            local_dir=local_dir,
            token=hf_token,
            ignore_patterns=["*.bin", "training_args.bin", "optimizer.pt"],
        )
        logger.info(f"Model downloaded to {downloaded_path}")
        return downloaded_path
    except Exception as e:
        logger.warning(f"Private repo download failed: {e}")

    # Try public fallback
    if hf_repo != PUBLIC_FALLBACK_REPO:
        logger.info(f"Trying public base model: {PUBLIC_FALLBACK_REPO}...")
        try:
            downloaded_path = snapshot_download(
                repo_id=PUBLIC_FALLBACK_REPO,
                local_dir=local_dir,
                token=hf_token,
            )
            logger.info(f"Base model downloaded to {downloaded_path}")
            logger.warning(
                "Using public base SparkTTS model (no custom speakers). "
                "For full speaker support, download your fine-tuned model."
            )
            return downloaded_path
        except Exception as e2:
            logger.warning(f"Public model download also failed: {e2}")

    # All downloads failed - provide manual instructions
    logger.error(
        "\nModel download failed. Options:\n"
        "  1. Download from Modal volume:\n"
        "     modal volume get veena3-models spark_tts_4speaker models/spark_tts_4speaker\n\n"
        "  2. Download from HuggingFace (if you have access):\n"
        f"     huggingface-cli download {PRIVATE_HF_REPO} --local-dir {local_dir}\n\n"
        "  3. Use base SparkTTS model:\n"
        f"     huggingface-cli download {PUBLIC_FALLBACK_REPO} --local-dir {local_dir}\n\n"
        "  4. Specify a custom path:\n"
        "     python -m veena3modal.local_server --model-path /path/to/model"
    )
    sys.exit(1)

def resolve_model_paths(model_path: str) -> tuple[str, str]:
    """
    Resolve LLM and BiCodec paths from a Spark TTS model directory.

    Spark TTS model structure (from SparkAudio/Spark-TTS-0.5B):
      model_path/
        LLM/          <- Language model (vLLM loads this)
        BiCodec/      <- Audio tokenizer/decoder
        config.yaml   <- Top-level config

    Fine-tuned models may have LLM files at root (flat structure).
    This function auto-detects both layouts.

    Returns:
        (llm_path, bicodec_path) resolved paths
    """
    # Check for subdirectory layout (SparkAudio/Spark-TTS-0.5B style)
    llm_subdir = os.path.join(model_path, "LLM")
    bicodec_subdir = os.path.join(model_path, "BiCodec")

    if os.path.isdir(llm_subdir) and os.path.exists(os.path.join(llm_subdir, "config.json")):
        llm_path = llm_subdir
    else:
        # Flat layout: LLM files at root
        llm_path = model_path

    # BiCodec path is always the root (it contains BiCodec/ subdir)
    bicodec_path = model_path

    logger.info(f"Resolved model paths:")
    logger.info(f"  LLM (vLLM):    {llm_path}")
    logger.info(f"  BiCodec:       {bicodec_path}")

    return llm_path, bicodec_path

def initialize_local_runtime(
    model_path: str,
    device: str = "cuda",
    gpu_memory_utilization: float = 0.85,
    enable_sr: bool = False,
    sr_checkpoint_dir: str | None = None,
    hf_token: str | None = None,
    num_engines: int = 1,
):
    """
    Initialize TTS runtime for local serving.

    This replaces Modal's @modal.enter lifecycle hook.
    Loads: vLLM engine, BiCodec decoder, prompt builder, pipelines.
    """
    from veena3modal.services.tts_runtime import initialize_runtime

    # Resolve LLM vs BiCodec paths from model directory
    llm_path, bicodec_path = resolve_model_paths(model_path)

    start = time.time()
    logger.info("=" * 60)
    logger.info("Initializing Veena3 TTS Runtime (local)")
    logger.info(f"  Model:       {model_path}")
    logger.info(f"  LLM path:    {llm_path}")
    logger.info(f"  BiCodec:     {bicodec_path}")
    logger.info(f"  Device:      {device}")
    logger.info(f"  GPU Memory:  {gpu_memory_utilization:.0%} (total)")
    logger.info(f"  Engines:     {num_engines}")
    logger.info(f"  SR:          {'enabled' if enable_sr else 'disabled'}")
    logger.info("=" * 60)

    runtime = initialize_runtime(
        model_path=llm_path,
        bicodec_path=bicodec_path,
        sr_checkpoint_dir=sr_checkpoint_dir if enable_sr else None,
        device=device,
        hf_token=hf_token,
        gpu_memory_utilization=gpu_memory_utilization,
        enable_sr=enable_sr,
        num_engines=num_engines,
    )

    elapsed = time.time() - start
    logger.info(f"Runtime initialized in {elapsed:.1f}s")
    logger.info(f"  Model version: {runtime.model_version}")
    logger.info(f"  Streaming: {'available' if runtime.streaming_pipeline else 'unavailable'}")
    logger.info(f"  Super-resolution: {'loaded' if runtime.sr_service and runtime.sr_service.is_loaded else 'disabled'}")

    return runtime

def create_local_app():
    """
    Create the FastAPI app for local serving.

    Same app factory as Modal deployment, but called from local context.
    """
    from veena3modal.api.fastapi_app import create_app
    return create_app()

def parse_args():
    """Parse CLI arguments for local server configuration."""
    parser = argparse.ArgumentParser(
        description="Veena3 TTS Local Server - GPU-accelerated text-to-speech",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python -m veena3modal.local_server                          # Auto-download model, serve on :8000
  python -m veena3modal.local_server --port 8080              # Custom port
  python -m veena3modal.local_server --model-path ./my_model  # Use local model
  python -m veena3modal.local_server --gpu-memory 0.5         # Limit GPU memory
  python -m veena3modal.local_server --workers 2              # Multiple workers (careful: each loads model)
        """,
    )

    # Model configuration
    model_group = parser.add_argument_group("Model")
    model_group.add_argument(
        "--model-path",
        type=str,
        default=os.environ.get("MODEL_PATH", DEFAULT_LOCAL_MODEL_DIR),
        help=f"Path to Spark TTS model (default: {DEFAULT_LOCAL_MODEL_DIR})",
    )
    model_group.add_argument(
        "--hf-repo",
        type=str,
        default=PRIVATE_HF_REPO,
        help=f"HuggingFace repo to download from (default: {PRIVATE_HF_REPO})",
    )
    model_group.add_argument(
        "--hf-token",
        type=str,
        default=os.environ.get("HF_TOKEN"),
        help="HuggingFace token (default: from HF_TOKEN env var)",
    )
    model_group.add_argument(
        "--skip-download",
        action="store_true",
        help="Don't auto-download model (fail if not found)",
    )

    # GPU configuration
    gpu_group = parser.add_argument_group("GPU")
    gpu_group.add_argument(
        "--device",
        type=str,
        default="cuda",
        choices=["cuda", "cpu"],
        help="Device for inference (default: cuda)",
    )
    gpu_group.add_argument(
        "--gpu-memory",
        type=float,
        default=float(os.environ.get("GPU_MEMORY_UTILIZATION", "0.25")),
        help="vLLM GPU memory utilization fraction (default: 0.25)",
    )
    gpu_group.add_argument(
        "--num-engines",
        type=int,
        default=int(os.environ.get("NUM_ENGINES", "1")),
        help="Number of vLLM engine instances (default: 1, use 2-3 for multi-engine mode)",
    )

    # Super-resolution
    sr_group = parser.add_argument_group("Super-Resolution")
    sr_group.add_argument(
        "--enable-sr",
        action="store_true",
        default=False,
        help="Enable AP-BWE super-resolution (16kHz -> 48kHz)",
    )
    sr_group.add_argument(
        "--sr-path",
        type=str,
        default=os.environ.get("AP_BWE_CHECKPOINT_DIR"),
        help="Path to AP-BWE checkpoint directory",
    )

    # Server configuration
    server_group = parser.add_argument_group("Server")
    server_group.add_argument(
        "--host",
        type=str,
        default="0.0.0.0",
        help="Bind address (default: 0.0.0.0)",
    )
    server_group.add_argument(
        "--port",
        type=int,
        default=int(os.environ.get("PORT", "8000")),
        help="Port to serve on (default: 8000)",
    )
    server_group.add_argument(
        "--workers",
        type=int,
        default=1,
        help="Number of uvicorn workers (default: 1, each loads full model)",
    )
    server_group.add_argument(
        "--reload",
        action="store_true",
        default=False,
        help="Enable auto-reload for development (incompatible with --workers > 1)",
    )

    # Auth
    auth_group = parser.add_argument_group("Auth")


**Inspecting GPU memory config**


sed -n '1,320p' veena3modal/core/constants.py


Output:
Chunk ID: 84b7e7
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1489
Output:
"""
Spark TTS Constants

Token IDs, special tokens, speakers, and emotions for Spark TTS model.
Migrated from Veena3/Orpheus to Spark TTS (BiCodec) architecture.
"""

import os

# Stop tokens for generation
# Spark TTS uses <|im_end|> as stop token
TRAINING_STOP_TOKEN_IDS = ["<|im_end|>"]  # String-based stop token for Spark TTS

# Spark TTS Speaker System - 12 Speakers from Training
# Maps user-facing speaker names to internal speaker IDs (speaker_0 to speaker_11)
# Model: BayAreaBoys/spark_tts_4speaker (HuggingFace) has 12 speaker tokens
SPEAKER_MAP = {
    "lipakshi": 0,    # speaker_0
    "vardan": 1,      # speaker_1
    "reet": 2,        # speaker_2
    "Nandini": 3,     # speaker_3
    "krishna": 4,     # speaker_4
    "anika": 5,       # speaker_5
    "adarsh": 6,      # speaker_6
    "Nilay": 7,       # speaker_7
    "Aarvi": 8,       # speaker_8
    "Asha": 9,        # speaker_9
    "Bittu": 10,      # speaker_10
    "Mira": 11,       # speaker_11
}

# Friendly speaker name mappings (user-facing → internal)
# Users can use these friendly names instead of internal names
FRIENDLY_SPEAKER_MAP = {
    "Mitra": "lipakshi",
    "Aaranya": "reet",
    "Taru": "Nandini",
    "Neer": "Nilay",
    "Dhruva": "vardan",
    "Ira": "anika",
    "Veda": "adarsh",
    "Aria": "krishna",
    # Direct mappings for new speakers
    "Aarvi": "Aarvi",
    "Asha": "Asha",
    "Bittu": "Bittu",
    "Mira": "Mira",
}

# Valid speaker names (for API validation)
# Include both internal names and friendly names
INDIC_SPEAKERS = list(SPEAKER_MAP.keys())
ALL_SPEAKER_NAMES = INDIC_SPEAKERS + list(FRIENDLY_SPEAKER_MAP.keys())

def resolve_speaker_name(name: str) -> str:
    """
    Resolve friendly speaker name to internal name.

    Args:
        name: Speaker name (friendly or internal)

    Returns:
        Internal speaker name

    Examples:
        resolve_speaker_name("Mitra") -> "lipakshi"
        resolve_speaker_name("lipakshi") -> "lipakshi"
    """
    # Check if it's a friendly name
    if name in FRIENDLY_SPEAKER_MAP:
        return FRIENDLY_SPEAKER_MAP[name]

    # Check if it's already an internal name
    if name in INDIC_SPEAKERS:
        return name

    # Invalid speaker name
    raise ValueError(f"Invalid speaker name: {name}. Valid names: {', '.join(ALL_SPEAKER_NAMES)}")

# Spark TTS Emotion Tags (bracket format: [emotion])
# NOTE: API users must now use [emotion] instead of <emotion>
INDIC_EMOTION_TAGS = [
    "[angry]",
    "[curious]",
    "[excited]",
    "[giggle]",
    "[laughs harder]",
    "[laughs]",
    "[screams]",
    "[sighs]",
    "[sings]",
    "[whispers]"
]

# Legacy emotion tags for backward compatibility mapping
# Maps old <emotion> format to new [emotion] format
LEGACY_EMOTION_MAP = {
    "<angry>": "[angry]",
    "<curious>": "[curious]",
    "<excited>": "[excited]",
    "<giggle>": "[giggle]",
    "<laugh_harder>": "[laughs harder]",
    "<laugh>": "[laughs]",
    "<scream>": "[screams]",
    "<sigh>": "[sighs]",
    "<sing>": "[sings]",
    "<whisper>": "[whispers]"
}

# All emotion tags (for validation)
ALL_EMOTION_TAGS = INDIC_EMOTION_TAGS

# Model configuration
DEFAULT_MODEL_PATH = "BayAreaBoys/spark_tts_4speaker"
DEFAULT_MAX_MODEL_LEN = 4096  # Lower than old model (was 8192)

# BiCodec Audio Tokenizer
# Location: resolved from env var MODEL_PATH or default
BICODEC_TOKENIZER_PATH = os.environ.get('MODEL_PATH', os.environ.get('SPARK_TTS_MODEL_PATH', '/models/spark_tts_4speaker'))

# Audio configuration
AUDIO_SAMPLE_RATE = 16000  # Changed from 24kHz to 16kHz for BiCodec
AUDIO_CHANNELS = 1
AUDIO_BITS_PER_SAMPLE = 16

# Generation defaults for Spark TTS
DEFAULT_TEMPERATURE = 0.8  # Higher than old model (was 0.4)
DEFAULT_TOP_K = 50
DEFAULT_TOP_P = 1.0
DEFAULT_MAX_TOKENS = 4096  # Increased to handle complex multilingual generation
DEFAULT_MIN_TOKENS = 28  # Minimum tokens for generation (legacy compatibility)
DEFAULT_REPETITION_PENALTY = 1.0  # Repetition penalty (legacy compatibility)
DEFAULT_SEED = None  # None = random, set integer for reproducibility

# vLLM Configuration optimizations for Spark TTS
# NOTE: gpu_memory_utilization adjusted based on available memory
# Production: 0.85, Limited memory: 0.3-0.5
#
# OPTIMIZATION Dec 2025:
# - enable_chunked_prefill: Prevents long prompts from blocking concurrent streams
# - async_scheduling: DISABLED - caused EngineCore issues in production
# OPTIMIZATION Feb 2026:
# Reduced gpu_memory_utilization from 0.85 to 0.25
# Model is only 0.5B (~1.3GB weights). At 0.85, vLLM pre-allocated 65GB for KV cache
# (enough for 1,399 concurrent seqs) when peak actual need is ~100 concurrent.
# At 0.25: ~18GB KV cache, supports ~380 concurrent seqs (4x peak), frees ~48GB VRAM.
# This enables fitting on L4 (24GB) or even T4 (16GB) GPUs.
VLLM_CONFIG = {
    "dtype": "bfloat16",
    "gpu_memory_utilization": 0.25,  # 0.5B model needs <2GB; 0.25 gives ~380 concurrent seqs
    "max_model_len": 4096,
    "max_num_batched_tokens": 4096,
    "enable_prefix_caching": True,  # Cache common prompts for faster TTFB
    "enable_chunked_prefill": True,  # Chunk long prefills to avoid blocking concurrent streams
    "enforce_eager": False,  # Enable CUDA graphs for low latency
    "disable_log_stats": False,
    "trust_remote_code": True,
    "tensor_parallel_size": 1,
}

# Streaming configuration (may need adjustment for BiCodec)
STREAM_BUFFER_SIZE = 50  # Buffer size for streaming (BiCodec-specific, TBD)
BICODEC_BATCH_SIZE = 64
BICODEC_BATCH_TIMEOUT_MS = 15

# Backward compatibility: Old SNAC constants for files not yet migrated
# These are NOT used by Spark TTS but needed by legacy streaming code
# TODO: Remove once streaming_pipeline.py is fully migrated to BiCodec
CODE_START_TOKEN_ID = 128257  # Legacy SNAC constant
CODE_END_TOKEN_ID = 128258    # Legacy SNAC constant
SNAC_MIN_ID = 128266          # Legacy SNAC constant
SNAC_MAX_ID = 156937          # Legacy SNAC constant


**Investigating prompt builder size**


sed -n '1,320p' veena3modal/processing/prompt_builder.py


Output:
Chunk ID: 3a4490
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2620
Output:
"""
Spark TTS Indic Prompt Builder

Builds prompts for Spark TTS with Indic speaker system.
Format: <|task_controllable_tts|><|start_content|>...<|end_content|>...

Migrated from Orpheus to Spark TTS architecture.
"""

import re
from typing import List, Tuple

from veena3modal.core.constants import INDIC_SPEAKERS, INDIC_EMOTION_TAGS, SPEAKER_MAP
from veena3modal.processing.emotion_normalizer import normalize_emotion_tags

class IndicPromptBuilder:
    """
    Builds prompts in Spark TTS format with Indic speaker system.

    Format:
    <|task_controllable_tts|>
    <|start_content|>{text with [emotions]}<|end_content|>
    <|start_style_label|><|speaker_{id}|><|end_style_label|>
    <|start_global_token|>

    CRITICAL: This format MUST match Spark TTS training exactly.

    Model Details:
    - HuggingFace: BayAreaBoys/spark_tts_4speaker
    - Architecture: Qwen2ForCausalLM with BiCodec audio tokenizer
    - Languages: Telugu, Hindi, English, and more
    - Speakers: 12 predefined (case-sensitive!) - must match training map
    - Emotions: 10 tags in [bracket] format
    """

    def __init__(self, tokenizer, model=None):
        """
        Initialize Indic prompt builder for Spark TTS.

        Args:
            tokenizer: Transformers tokenizer
            model: Model instance (optional, kept for backward compatibility)
        """
        self.tokenizer = tokenizer
        self.model = model

    def build_prefix(
        self,
        speaker: str,
        text: str,
        validate: bool = True
    ) -> str:
        """
        Build Spark TTS format prompt with Indic speaker.

        Args:
            speaker: Speaker name (one of 12 predefined speakers, case-sensitive!)
                Examples: 'lipakshi', 'reet', 'Nandini', 'Nilay', 'vardan', 'anika', 'adarsh',
                'krishna', 'Aarvi', 'Asha', 'Bittu', 'Mira'
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome to the demo."
                - "[laughs] The results were amazing!"
                - "The results were amazing and then [giggle] we celebrated!"
                - "नमस्ते! [excited] आज का दिन बहुत अच्छा है।" (Hindi)
                - "[curious] మీరు ఎలా ఉన్నారు?" (Telugu)
                - "Hello <laugh> this works too!" (will be normalized to [laughs])
            validate: Check format correctness (recommended for first use)

        Returns:
            Formatted prompt string ready for tokenization
            Format: <|task_controllable_tts|><|start_content|>text<|end_content|>...

        Note: Emotion tags should be inline in text. The API will normalize:
        - <laugh> → [laughs] (old format)
        - laughing → [laughs] (natural language)
        """
        # Normalize emotion tags: <emotion> → [emotion] (legacy compatibility)
        text = normalize_emotion_tags(text)

        # Validate if requested
        if validate:
            self._validate_inputs(speaker, text)

        # Get speaker ID from mapping
        speaker_id = SPEAKER_MAP.get(speaker)
        if speaker_id is None:
            raise ValueError(f"Invalid speaker: {speaker}. Valid speakers: {list(SPEAKER_MAP.keys())}")

        # Build Spark TTS format prompt
        prompt = "".join([
            "<|task_controllable_tts|>",
            "<|start_content|>",
            text,
            "<|end_content|>",
            "<|start_style_label|>",
            f"<|speaker_{speaker_id}|>",
            "<|end_style_label|>",
            "<|start_global_token|>"
        ])

        return prompt

    def build_prefix_ids(
        self,
        speaker: str,
        text: str,
        validate: bool = True
    ) -> List[int]:
        """
        Build prefix as token IDs (for testing/debugging).

        Args:
            speaker: Speaker name
            text: Text to synthesize (with inline emotions)
            validate: Check format correctness

        Returns:
            List of token IDs for the prefix
        """
        prompt = self.build_prefix(speaker, text, validate)
        token_ids = self.tokenizer.encode(prompt, add_special_tokens=False)
        return token_ids

    def _validate_inputs(self, speaker: str, text: str):
        """
        Validate speaker and text format.

        Args:
            speaker: Speaker name
            text: Text to synthesize (with inline emotions)

        Raises:
            ValueError: If inputs are invalid
        """
        # Validate speaker (case-sensitive!)
        if speaker not in INDIC_SPEAKERS:
            raise ValueError(
                f"Invalid speaker '{speaker}'. Must be one of (case-sensitive): "
                f"{', '.join(INDIC_SPEAKERS)}"
            )

        # Check text isn't empty
        if not text.strip():
            raise ValueError("Text cannot be empty")

        # Validate inline emotion tags in text (after normalization)
        self._validate_emotion_tags(text)

    def _validate_emotion_tags(self, text: str):
        """
        Validate emotion tags in text are in the allowed set for Spark TTS.

        Warns if unknown emotion tags are found (won't break, but won't work).

        Args:
            text: Text with potential emotion tags
        """
        # Find all [emotion] tags in text (Spark TTS format)
        emotion_pattern = r'\[([a-z\s]+)\]'
        found_emotions = re.findall(emotion_pattern, text)

        if not found_emotions:
            return  # No emotion tags, nothing to validate

        # Check each found emotion
        unknown_tags = []
        for emotion in found_emotions:
            tag = f'[{emotion}]'
            if tag not in INDIC_EMOTION_TAGS:
                unknown_tags.append(tag)

        if unknown_tags:
            print(f"⚠  Warning: Found {len(unknown_tags)} unknown emotion tag(s) for Spark TTS:")
            for tag in unknown_tags:
                print(f"    - {tag} (will be treated as regular text)")
            print(f"\n📋 Valid Spark TTS emotion tags:")
            for tag in INDIC_EMOTION_TAGS:
                print(f"    {tag}")
            print("\n")

    def extract_emotion_tags(self, text: str) -> List[Tuple[str, int]]:
        """
        Extract emotion tags and their positions from text.

        Args:
            text: Text with emotion tags

        Returns:
            List of (emotion_tag, position) tuples
        """
        emotion_pattern = r'\[([a-z\s]+)\]'
        matches = []

        for match in re.finditer(emotion_pattern, text):
            tag = match.group(0)
            if tag in INDIC_EMOTION_TAGS:
                matches.append((tag, match.start()))

        return matches

    def build_prefix_with_globals(
        self,
        speaker: str,
        text: str,
        global_ids: List[int],
        validate: bool = True
    ) -> str:
        """
        Build prompt with pre-generated global tokens for voice consistency in chunked generation.

        This is critical for multi-chunk text processing: the 32 global tokens encode
        speaker identity (voice DNA). By injecting the same global tokens from the first
        chunk into subsequent chunks, we ensure consistent voice across the entire text.

        Use Case:
        - Chunk 1: Use build_prefix() → model generates 32 global tokens + semantic tokens
        - Chunk 2+: Use build_prefix_with_globals() with captured global tokens
                    → model skips global generation, generates only semantic tokens

        Args:
            speaker: Speaker name (same as first chunk)
            text: Text chunk to synthesize
            global_ids: List of 32 global token IDs captured from first chunk
            validate: Check format correctness

        Returns:
            Formatted prompt with pre-filled global tokens

        Raises:
            ValueError: If global_ids doesn't contain exactly 32 tokens

        Thread Safety:
            This method is stateless and thread-safe. Global tokens are passed
            explicitly per-request, not stored in any shared state.
        """
        # Validate global tokens count
        EXPECTED_GLOBAL_COUNT = 32
        if len(global_ids) != EXPECTED_GLOBAL_COUNT:
            raise ValueError(
                f"Expected exactly {EXPECTED_GLOBAL_COUNT} global tokens, got {len(global_ids)}. "
                f"This likely indicates an issue with first chunk generation."
            )

        # Normalize emotion tags: <emotion> → [emotion] (legacy compatibility)
        text = normalize_emotion_tags(text)

        # Validate if requested
        if validate:
            self._validate_inputs(speaker, text)

        # Get speaker ID from mapping
        speaker_id = SPEAKER_MAP.get(speaker)
        if speaker_id is None:
            raise ValueError(f"Invalid speaker: {speaker}. Valid speakers: {list(SPEAKER_MAP.keys())}")

        # Build global tokens string: <|bicodec_global_0|><|bicodec_global_1|>...<|bicodec_global_31|>
        global_tokens_str = "".join([f"<|bicodec_global_{gid}|>" for gid in global_ids])

        # Build Spark TTS format prompt WITH pre-filled global tokens
        # The model will see these and continue with semantic token generation only
        prompt = "".join([
            "<|task_controllable_tts|>",
            "<|start_content|>",
            text,
            "<|end_content|>",
            "<|start_style_label|>",
            f"<|speaker_{speaker_id}|>",
            "<|end_style_label|>",
            "<|start_global_token|>",
            global_tokens_str,  # Pre-filled 32 global tokens
            "<|start_semantic_token|>"  # Signal to start semantic tokens directly
        ])

        return prompt

    @staticmethod
    def get_available_speakers() -> List[str]:
        """Get list of available speakers."""
        return INDIC_SPEAKERS.copy()

    @staticmethod
    def get_available_emotions() -> List[str]:
        """Get list of available emotions (without [ ])."""
        return [e.strip('[]') for e in INDIC_EMOTION_TAGS]


**Identifying missing max_tokens causing timeouts**


I found one concrete issue in the HTTP stress script: it never sends `max_tokens`, so the server falls back to `4096` per request. I’m now running a single direct-runtime probe with low `max_tokens` to confirm whether generation itself works and to isolate the remaining stall.


python - <<'PY'
import asyncio, time, os
from veena3modal.local_server import resolve_model_paths, DEFAULT_LOCAL_MODEL_DIR
from veena3modal.services.tts_runtime import initialize_runtime, generate_speech, is_initialized

model_path = DEFAULT_LOCAL_MODEL_DIR
llm,bicodec = resolve_model_paths(model_path)
if not is_initialized():
    initialize_runtime(model_path=llm,bicodec_path=bicodec,device='cuda',gpu_memory_utilization=0.25)

async def main():
    t0=time.time()
    audio, metrics = await generate_speech(
        text='hello',
        speaker='lipakshi',
        max_tokens=64,
        temperature=0.2,
        top_k=40,
        top_p=0.95,
    )
    print('elapsed', time.time()-t0)
    print('audio', None if audio is None else len(audio))
    print('keys', sorted(metrics.keys()))
    for k in ['generation_ms','llm_token_total','llm_batch_count','llm_generation_wall_ms','llm_batch_wall_ms_total','llm_batch_gpu_ms_total','llm_time_per_token_ms','bicodec_decode_wall_ms']:
        print(k, metrics.get(k))

asyncio.run(main())
PY


Output:
Chunk ID: 163e53
Wall time: 0.0503 seconds
Process exited with code 127
Original token count: 12
Output:
/bin/bash: line 1: python: command not found


The shell only has `python3`, so I’m rerunning the direct probe with that and capturing full generation timing fields.


python3 - <<'PY'
import asyncio, time
from veena3modal.local_server import resolve_model_paths, DEFAULT_LOCAL_MODEL_DIR
from veena3modal.services.tts_runtime import initialize_runtime, generate_speech, is_initialized

model_path = DEFAULT_LOCAL_MODEL_DIR
llm,bicodec = resolve_model_paths(model_path)
if not is_initialized():
    initialize_runtime(model_path=llm,bicodec_path=bicodec,device='cuda',gpu_memory_utilization=0.25)

async def main():
    t0=time.time()
    audio, metrics = await generate_speech(
        text='hello',
        speaker='lipakshi',
        max_tokens=64,
        temperature=0.2,
        top_k=40,
        top_p=0.95,
    )
    print('elapsed', time.time()-t0)
    print('audio', None if audio is None else len(audio))
    print('gen', metrics.get('generation_ms'))
    print('tok', metrics.get('llm_token_total'))
    print('batches', metrics.get('llm_batch_count'))
    print('llm_generation_wall_ms', metrics.get('llm_generation_wall_ms'))
    print('llm_batch_wall_ms_total', metrics.get('llm_batch_wall_ms_total'))
    print('llm_batch_gpu_ms_total', metrics.get('llm_batch_gpu_ms_total'))
    print('time_per_token', metrics.get('llm_time_per_token_ms'))
    print('bicodec_decode_wall_ms', metrics.get('bicodec_decode_wall_ms'))

asyncio.run(main())
PY


Output:
Chunk ID: 0f1b40
Wall time: 1.0015 seconds
Process running with session ID 45543
Original token count: 0
Output:


**Planning session polling**


Output:
Chunk ID: 1290ba
Wall time: 5.0005 seconds
Process running with session ID 45543
Original token count: 1277
Output:
[2026-02-17 04:25:16] INFO tts_runtime.py:148: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.25
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-17 04:25:17 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 04:25:17 [model.py:1561] Using max model len 4096
INFO 02-17 04:25:17 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 04:25:17 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:18 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}


**Considering retry timing**


Output:
Chunk ID: 6e2ab7
Wall time: 5.0013 seconds
Process running with session ID 45543
Original token count: 437
Output:
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:20 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:58335 backend=nccl
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:20 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:21 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:22 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=3808028) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=3808028) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=3808028)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=3808028)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.33s/it]
(EngineCore_DP0 pid=3808028)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.33s/it]
(EngineCore_DP0 pid=3808028)
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:23 [default_loader.py:291] Loading weights took 1.40 seconds
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:24 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.949503 seconds


**Continuing polling process**


Output:
Chunk ID: 318e7a
Wall time: 5.0017 seconds
Process running with session ID 45543
Original token count: 490
Output:
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:28 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:28 [backends.py:872] Dynamo bytecode transform time: 4.26 s
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:31 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.650 s
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:31 [monitor.py:34] torch.compile takes 4.91 s in total
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:32 [gpu_worker.py:356] Available KV cache memory: 18.06 GiB
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:32 [kv_cache_utils.py:1307] GPU KV cache size: 1,577,712 tokens
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:32 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 385.18x
(EngineCore_DP0 pid=3808028)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 32.80it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 35.43it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 36.61it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 37.02it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 37.18it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 37.02it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 36.68it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 35.08it/s]


Output:
Chunk ID: f6d787
Wall time: 5.0013 seconds
Process running with session ID 45543
Original token count: 371
Output:

Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:00<00:00, 35.18it/s]
(EngineCore_DP0 pid=3808028)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 29.25it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 33.56it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 35.02it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 35.54it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 36.10it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.21it/s]
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:35 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=3808028) INFO 02-17 04:25:35 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.79 seconds
[2026-02-17 04:25:36] INFO tts_runtime.py:156: Initializing prompt builder...
[2026-02-17 04:25:36] INFO tts_runtime.py:163: Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


**Monitoring codec loading duration**


Output:
Chunk ID: 21cbb4
Wall time: 5.0010 seconds
Process running with session ID 45543
Original token count: 366
Output:
[2026-02-17 04:25:43] INFO tts_runtime.py:170: Initializing TTS pipeline...
[2026-02-17 04:25:43] INFO tts_runtime.py:181: Initializing streaming pipeline...
[2026-02-17 04:25:43] INFO tts_runtime.py:231: ✅ TTS runtime initialized in 32100ms
[2026-02-17 04:25:43] INFO tts_runtime.py:232:    Model version: LLM
[2026-02-17 04:25:43] INFO tts_runtime.py:277: Pre-computing global tokens for all speakers...
[2026-02-17 04:25:43] INFO tts_runtime.py:321:   Cached globals for lipakshi
[2026-02-17 04:25:44] INFO tts_runtime.py:321:   Cached globals for vardan
[2026-02-17 04:25:44] INFO tts_runtime.py:321:   Cached globals for reet
[2026-02-17 04:25:45] INFO tts_runtime.py:321:   Cached globals for Nandini
[2026-02-17 04:25:45] INFO tts_runtime.py:321:   Cached globals for krishna
[2026-02-17 04:25:45] INFO tts_runtime.py:321:   Cached globals for anika
[2026-02-17 04:25:46] INFO tts_runtime.py:321:   Cached globals for adarsh
[2026-02-17 04:25:46] INFO tts_runtime.py:321:   Cached globals for Nilay
[2026-02-17 04:25:47] INFO tts_runtime.py:321:   Cached globals for Aarvi
[2026-02-17 04:25:47] INFO tts_runtime.py:321:   Cached globals for Asha
[2026-02-17 04:25:47] INFO tts_runtime.py:321:   Cached globals for Bittu
[2026-02-17 04:25:48] INFO tts_runtime.py:321:   Cached globals for Mira
[2026-02-17 04:25:48] INFO tts_runtime.py:344: Speaker globals cached: 12/12 speakers in 4891ms
[2026-02-17 04:25:48] INFO pipeline.py:182: perf_non_stream_start


**Assessing non-streaming hang**


Output:
Chunk ID: b7c29a
Wall time: 5.0005 seconds
Process running with session ID 45543
Original token count: 0
Output:


Summary: auto


**Extending poll interval**


Output:
Chunk ID: 3fb577
Wall time: 30.0008 seconds
Process running with session ID 45543
Original token count: 0
Output:


**Adjusting request polling timeout**


Output:
Chunk ID: 53b7c4
Wall time: 30.0015 seconds
Process running with session ID 45543
Original token count: 0
Output:


**Debugging asynchronous hang**


The direct probe reproduced the stall: runtime initializes and logs `perf_non_stream_start`, then no first-batch event arrives even after a long wait. I’m now running low-level vLLM probes to isolate whether the hang is in sampling config, stop criteria, or engine scheduling.


ps -eo pid,ppid,cmd | rg "python3 -|EngineCore|vllm|veena3modal" -n


Output:
Chunk ID: 2180ea
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 5
Output:
0, 23441, 81920, 29


Output:
Chunk ID: 1a197d
Wall time: 0.0517 seconds
Process exited with code 0
Original token count: 389
Output:
304:3807901 3675569 /bin/bash -c python3 - <<'PY' import asyncio, time from veena3modal.local_server import resolve_model_paths, DEFAULT_LOCAL_MODEL_DIR from veena3modal.services.tts_runtime import initialize_runtime, generate_speech, is_initialized  model_path = DEFAULT_LOCAL_MODEL_DIR llm,bicodec = resolve_model_paths(model_path) if not is_initialized():     initialize_runtime(model_path=llm,bicodec_path=bicodec,device='cuda',gpu_memory_utilization=0.25)  async def main():     t0=time.time()     audio, metrics = await generate_speech(         text='hello',         speaker='lipakshi',         max_tokens=64,         temperature=0.2,         top_k=40,         top_p=0.95,     )     print('elapsed', time.time()-t0)     print('audio', None if audio is None else len(audio))     print('gen', metrics.get('generation_ms'))     print('tok', metrics.get('llm_token_total'))     print('batches', metrics.get('llm_batch_count'))     print('llm_generation_wall_ms', metrics.get('llm_generation_wall_ms'))     print('llm_batch_wall_ms_total', metrics.get('llm_batch_wall_ms_total'))     print('llm_batch_gpu_ms_total', metrics.get('llm_batch_gpu_ms_total'))     print('time_per_token', metrics.get('llm_time_per_token_ms'))     print('bicodec_decode_wall_ms', metrics.get('bicodec_decode_wall_ms'))  asyncio.run(main()) PY
305:3807902 3807901 python3 -
306:3808028 3807902 VLLM::EngineCore
314:3808733 3675569 /bin/bash -c ps -eo pid,ppid,cmd | rg "python3 -|EngineCore|vllm|veena3modal" -n
316:3808735 3808733 rg python3 -|EngineCore|vllm|veena3modal -n


**Identifying async scheduling bug**


Output: write_stdin failed: stdin is closed for this session; rerun exec_command with tty=true to keep stdin open


**Confirming process persistence after input close**


ps -eo pid,ppid,cmd | rg "3807902|3808028|python3 - <<'PY'|VLLM::EngineCore" -n