Output:
Chunk ID: 77dd84
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 2469
Output:
vast_docs/pricing.md
vast_docs/pyworker.md
vast_docs/custom_pyworkers.md
vast_docs/performance_testing.md
vast_docs/serverless_overview.md
vast_docs/debugging.md
vast_docs/serverless_params.md
vast_docs/architecture.md
vast_docs/quickstart.md
vast_docs/inside_serverless_gpu.md
vast_docs/creating_endpoints_workergroups.md
vast_docs/route.md
vast_docs/logs.md
vast_docs/worker_list.md
production/worker.py
production/test_stress_full.py
production/test_stress.py
production/server.py
production/test_smoke.py
production/Dockerfile
benchmark_results/ws_debug_quick.json
benchmark_results/capacity_c50_s45_health.csv
production/Dockerfile.slim
production/onstart.sh
production/requirements.txt
production/deploy.py
production/__pycache__/server.cpython-310.pyc
production/__pycache__/test_stress_full.cpython-310.pyc
production/__pycache__/test_stress.cpython-310.pyc
production/__pycache__/deploy.cpython-310.pyc
production/__pycache__/worker.cpython-310.pyc
production/__pycache__/test_smoke.cpython-310.pyc
benchmark_results/fleurs_hindi_comparison.json
benchmark_results/capacity_c250_s60_streams.csv
benchmark_results/capacity_c20_s45_streams.csv
benchmark_results/stress_c50_s2.0_delays.csv
benchmark_results/stress_results_20260209_202718.json
benchmark_results/stress_c250_s2.0.json
benchmark_results/fleurs_hindi_details.csv
benchmark_results/stress_c1000_s2.0.json
benchmark_results/capacity_c80_s60.json
benchmark_results/indicvoices_hindi_summary.json
benchmark_results/stress_results_20260209_202433.csv
benchmark_results/capacity_c200_s60_health.csv
benchmark_results/ws_v13_fleurs_chunk560_c25_75.json
benchmark_results/sustained_v10_sse_c75_r3.csv
benchmark_results/capacity_c150_s60_streams.csv
benchmark_results/stress_c50_s2.0_streams.csv
benchmark_results/capacity_c100_s60.json
benchmark_results/stress_c500_s2.0_streams.csv
benchmark_results/sustained_v10_sse_c50_r6.csv
benchmark_results/stress_results_20260209_201434.json
benchmark_results/fleurs_hindi_summary.json
benchmark_results/sanity_full_sse_c1.csv
benchmark_results/capacity_c140_s60_health.csv
benchmark_results/sustained_v10_http_c100_r2_recheck.csv
benchmark_results/stress_c50_s2.0_health.csv
benchmark_results/capacity_c1_s60.json
benchmark_results/stress_results_20260209_201434.csv
benchmark_results/overall_concurrent_p99.json
benchmark_results/scaling_law_v10_20260209.json
benchmark_results/full_v10_fleurs_c1_100_r2_details.csv
benchmark_results/capacity_c120_s60.json
benchmark_results/sanity_full_http_c1.csv
benchmark_results/capacity_c1_sustained15_delays.csv
benchmark_results/capacity_c250_s60.json
benchmark_results/ws_v13_max500_fleurs_c127_r3.csv
benchmark_results/loadtest_concurrent_5_details.csv
benchmark_results/capacity_c110_s60_streams.csv
benchmark_results/capacity_c50_s45.json
benchmark_results/capacity_c20_s60.json
benchmark_results/stress_sweep_s2.0.json
benchmark_results/stress_c250_s2.0_delays.csv
benchmark_results/ws_v13_fleurs_chunk320_c25_75.csv
benchmark_results/capacity_c10_s60_health.csv
benchmark_results/capacity_c110_s60_health.csv
benchmark_results/capacity_c150_s60.json
benchmark_results/capacity_c120_s60_streams.csv
benchmark_results/sustained_v10_sse_c100_r1_details.csv
benchmark_results/sustained_v10_sse_c100_r2_recheck2_details.csv
benchmark_results/sustained_v10_sse_c50_r6_details.csv
benchmark_results/capacity_c1_s45_streams.csv
benchmark_results/sanity_full_http_c1_r10_details.csv
benchmark_results/capacity_c5_sustained20_streams.csv
benchmark_results/capacity_c80_s60_health.csv
benchmark_results/ws_v13_fleurs_c1_25.csv
benchmark_results/capacity_c80_s45_health.csv
benchmark_results/capacity_c30_s45.json
benchmark_results/capacity_c5_sustained45_health.csv
benchmark_results/stress_c50_s2.0.json
benchmark_results/stress_c1000_s2.0_health.csv
benchmark_results/loadtest_concurrent_10_summary.json
benchmark_results/capacity_c80_s45.json
benchmark_results/capacity_c140_s60.json
benchmark_results/capacity_c100_s60_health.csv
benchmark_results/capacity_c30_s45_streams.csv
benchmark_results/capacity_c175_s60.json
benchmark_results/sustained_v10_sse_c100_r2_recheck2.csv
benchmark_results/capacity_c200_s60_streams.csv
benchmark_results/full_v10_fleurs_c1_100_r2.json
benchmark_results/gpu_ceiling_test.json
benchmark_results/ws_v11_quick.csv
benchmark_results/stress_results_20260209_202718.csv
benchmark_results/capacity_c120_s60_health.csv
benchmark_results/sanity_v10_http_c1_r10.json
benchmark_results/sustained_v10_http_c50_r6_details.csv
benchmark_results/ws_v13_fleurs_c127.json
benchmark_results/loadtest_concurrent_50_summary.json
benchmark_results/capacity_c1_sustained15_streams.csv
benchmark_results/local_concurrent_proper.json
benchmark_results/indicvoices_hindi_details.csv
benchmark_results/capacity_c100_s45.json
benchmark_results/stress_c5_s2.0_delays.csv
benchmark_results/ws_fleurs_sample_meta.json
benchmark_results/capacity_c10_s45.json
benchmark_results/capacity_c10_s45_streams.csv
benchmark_results/ws_v13_fleurs_c127.csv
benchmark_results/stress_c10_s2.0_delays.csv
benchmark_results/stress_c500_s2.0_health.csv
benchmark_results/capacity_c5_sustained45_delays.csv
benchmark_results/capacity_c5_s45_health.csv
benchmark_results/stress_c1_s1.0.json
benchmark_results/stress_c100_s2.0.json
benchmark_results/stress_c500_s2.0_delays.csv
benchmark_results/stress_c250_s2.0_health.csv
benchmark_results/stress_c500_s2.0.json
benchmark_results/stress_results_20260209_195730.json
benchmark_results/local_direct_latency.json
benchmark_results/capacity_c200_s60.json
benchmark_results/ws_v13_max500_fleurs_c127_r2.json
benchmark_results/capacity_c5_s45_streams.csv
benchmark_results/capacity_c1_s60_streams.csv
benchmark_results/full_v13_fleurs_c1_50.csv
benchmark_results/capacity_c5_sustained30_streams.csv
benchmark_results/stress_sweep_h100_s2.0.json
benchmark_results/capacity_c5_sustained60_streams.csv
benchmark_results/capacity_c130_s60.json
benchmark_results/capacity_c1_s45_health.csv
benchmark_results/sanity_full_http_c1_r10.csv
benchmark_results/loadtest_concurrent_50_details.csv
benchmark_results/capacity_c10_s45_health.csv
benchmark_results/capacity_c140_s60_streams.csv
benchmark_results/sustained_v10_http_c50_r6.csv
benchmark_results/capacity_c100_s60_streams.csv
benchmark_results/stress_c10_s2.0_streams.csv
benchmark_results/capacity_c5_sustained45_streams.csv
benchmark_results/capacity_c5_sustained20.json
benchmark_results/capacity_c150_s60_health.csv
benchmark_results/capacity_c50_s60_health.csv
benchmark_results/sustained_v10_sse_c75_r3_details.csv
benchmark_results/capacity_c20_s45_health.csv
benchmark_results/stress_c100_s2.0_health.csv
benchmark_results/sustained_v10_sse_c100_r2_recheck.json
benchmark_results/ws_v12_fleurs_c1_25.json
benchmark_results/capacity_c5_sustained60_health.csv
benchmark_results/sanity_v10_sse_c1_r10.json
benchmark_results/sanity_full_http_c1_r10.json
benchmark_results/ws_v13_fleurs_c50_100.json
benchmark_results/stress_c1_s1.0_streams.csv
benchmark_results/capacity_c175_s60_health.csv
benchmark_results/ws_v13_max500_fleurs_c127.json
benchmark_results/stress_c1_s1.0_delays.csv
benchmark_results/capacity_c100_s45_streams.csv
benchmark_results/sustained_v10_sse_c100_r1.csv
benchmark_results/sanity_v10_sse_c1_r10_details.csv
benchmark_results/capacity_c110_s60.json
benchmark_results/stress_c100_s2.0_streams.csv
benchmark_results/sustained_v10_http_c100_r2_recheck_details.csv
benchmark_results/capacity_c300_s60_streams.csv
benchmark_results/ws_v11_fleurs_c1_25.json
benchmark_results/sustained_v10_sse_c100_r2_recheck.csv
benchmark_results/capacity_c300_s60_health.csv
benchmark_results/capacity_c30_s60_streams.csv
benchmark_results/capacity_c130_s60_streams.csv
benchmark_results/capacity_c50_s45_streams.csv
benchmark_results/capacity_c20_s45.json
benchmark_results/capacity_c100_s45_health.csv
benchmark_results/ws_v13_max500_fleurs_c100_r1.csv
benchmark_results/capacity_c5_sustained45.json
benchmark_results/sustained_v10_sse_c100_r2_recheck2.json
benchmark_results/sanity_full_sse_c1.json
benchmark_results/capacity_c1_s60_health.csv
benchmark_results/loadtest_concurrent_5_summary.json
benchmark_results/local_batched_sweep.json
benchmark_results/capacity_c1_s45.json
benchmark_results/sustained_v10_http_c50_r6.json
benchmark_results/capacity_c5_sustained30_delays.csv
benchmark_results/capacity_c20_s60_streams.csv
benchmark_results/stress_c5_s2.0_health.csv
benchmark_results/capacity_c5_sustained30_health.csv
benchmark_results/stress_c1_s2.0_delays.csv
benchmark_results/ws_v11_fleurs_c1_25.csv
benchmark_results/capacity_c175_s60_streams.csv
benchmark_results/stress_results_20260209_195631.json
benchmark_results/indicvoices_hindi_comparison.json
benchmark_results/stress_c10_s2.0_health.csv
benchmark_results/ws_v13_fleurs_chunk320_c25_75.json
benchmark_results/capacity_c5_sustained20_health.csv
benchmark_results/capacity_c5_s45.json
benchmark_results/capacity_c30_s45_health.csv
benchmark_results/loadtest_concurrent_1_summary.json
benchmark_results/sustained_v10_sse_c100_r2_recheck_details.csv
benchmark_results/stress_c1_s2.0_streams.csv
benchmark_results/capacity_c80_s60_streams.csv
benchmark_results/stress_results_20260209_195631.csv
benchmark_results/stress_c10_s2.0.json
benchmark_results/stress_c1_s2.0_health.csv
benchmark_results/stress_c250_s2.0_streams.csv
benchmark_results/ws_v13_max500_fleurs_c100_r1.json
benchmark_results/capacity_c50_s60.json
benchmark_results/ws_v13_max500_fleurs_c200_500.csv
benchmark_results/stress_c1000_s2.0_delays.csv
benchmark_results/stress_c5_s2.0_streams.csv
benchmark_results/stress_results_20260209_202433.json
benchmark_results/stress_c1_s1.0_health.csv
benchmark_results/capacity_c300_s60.json
benchmark_results/capacity_c5_s60.json
benchmark_results/sanity_full_http_c1.json
benchmark_results/ws_v12_fleurs_c1_25.csv
benchmark_results/capacity_c130_s60_health.csv


Output:
Chunk ID: 1a6299
Wall time: 0.0509 seconds
Process exited with code 128
Original token count: 18
Output:
fatal: not a git repository (or any of the parent directories): .git


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Inspecting deployment and test scripts**


I’ve confirmed the repo contains the full deployment/benchmark stack and existing results. Next I’m reading the deployment and stress harness code to run an H100-specific serverless benchmark with Hindi-model WER tracking and bottleneck diagnostics.


sed -n '1,320p' production/test_stress.py


sed -n '1,320p' production/test_stress_full.py


Output:
Chunk ID: 0da238
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 1509
Output:
#!/usr/bin/env python3
"""
Vast.ai Serverless Deployment for Hindi ASR.
Usage: python deploy.py {create|status|workers|logs|route|destroy}
"""
import os, sys, json, argparse, requests
sys.path.insert(0, os.path.dirname(__file__))
try:
    from dotenv import load_dotenv
    load_dotenv(os.path.join(os.path.dirname(__file__), "..", ".env"))
except ImportError:
    pass

VAST_API_KEY = os.environ.get("VAST_KEY", "")
VAST_API = "https://console.vast.ai/api/v0"
VAST_RUN = "https://run.vast.ai"
ENDPOINT_NAME = os.environ.get("VAST_ENDPOINT_NAME", "hindi-asr-prod")
DOCKER_IMAGE = os.environ.get("VAST_DOCKER_IMAGE", "bharathkumar192/hindi-nemotron-asr:v10-production")
TEMPLATE_HASH = os.environ.get("VAST_TEMPLATE_HASH", "")
LAUNCH_ARGS = os.environ.get("VAST_LAUNCH_ARGS", "-p 8000:8000")

ENDPOINT_CONFIG = {
    "endpoint_name": ENDPOINT_NAME,
    "min_load": 40,
    "target_util": 0.65,
    "cold_mult": 2.5,
    "cold_workers": 2,
    "max_workers": 16,
}

SEARCH_PARAMS = {
    "verified": {"eq": True}, "rentable": {"eq": True},
    "rented": {"eq": False}, "gpu_ram": {"gte": 40},
    "num_gpus": {"eq": 1}, "reliability": {"gte": 0.95},
    "inet_down": {"gte": 200},
}

def headers():
    return {"Accept": "application/json", "Content-Type": "application/json",
            "Authorization": f"Bearer {VAST_API_KEY}"}

def get_endpoints():
    try:
        r = requests.get(f"{VAST_API}/endptjobs/", headers=headers())
        data = r.json()
        if isinstance(data, dict):
            return data.get("results", [])
        if isinstance(data, list):
            return data
    except Exception:
        pass
    return []

def get_endpoint_info():
    for ep in get_endpoints():
        if ep.get("endpoint_name") == ENDPOINT_NAME:
            return ep
    return None

def create_endpoint():
    print(f"Creating endpoint: {ENDPOINT_NAME}")
    r = requests.post(f"{VAST_API}/endptjobs/", headers=headers(), json=ENDPOINT_CONFIG)
    d = r.json()
    if d.get("success"):
        print(f"  Created: id={d['result']}")
        return d["result"]
    print(f"  Error: {d}")
    return get_endpoint_id()

def get_endpoint_id():
    ep = get_endpoint_info()
    return ep.get("id") if ep else None

def get_endpoint_key():
    ep = get_endpoint_info()
    return ep.get("api_key") if ep else None

def get_template_hash():
    if TEMPLATE_HASH:
        return TEMPLATE_HASH
    # Try to infer from existing workergroup
    try:
        r = requests.get(f"{VAST_API}/workergroups/", headers=headers())
        data = r.json()
        results = data.get("results", []) if isinstance(data, dict) else data
        for wg in results or []:
            if wg.get("endpoint_name") == ENDPOINT_NAME or wg.get("endpoint_id") == get_endpoint_id():
                return wg.get("template_hash") or ""
    except Exception:
        pass
    return ""

def create_workergroup(eid=None):
    eid = eid or get_endpoint_id()
    if not eid: print("No endpoint found"); return
    print(f"Creating workergroup for endpoint {eid}")
    payload = {
        "endpoint_id": eid,
        "gpu_ram": 40,
    }
    template_hash = get_template_hash()
    if template_hash:
        payload["template_hash"] = template_hash
        payload["search_params"] = ""
    else:
        payload["search_params"] = json.dumps(SEARCH_PARAMS)
    if LAUNCH_ARGS:
        payload["launch_args"] = LAUNCH_ARGS
    r = requests.post(f"{VAST_API}/workergroups/", headers=headers(), json=payload)
    d = r.json()
    print(f"  {'Created: id=' + str(d['result']) if d.get('success') else 'Error: ' + str(d)}")

def get_workers():
    eid = get_endpoint_id()
    ekey = get_endpoint_key()
    if not eid or not ekey:
        return []
    r = requests.post(f"{VAST_RUN}/get_endpoint_workers/",
        json={"id": eid, "api_key": ekey},
        headers={"Content-Type": "application/json"})
    try:
        d = r.json()
        return d if isinstance(d, list) else d.get("workers", [])
    except: return []

def get_route(cost=10.0):
    ekey = get_endpoint_key()
    if not ekey:
        return {"error": "endpoint api_key not found"}
    r = requests.post(f"{VAST_RUN}/route/",
        json={"endpoint": ENDPOINT_NAME, "api_key": ekey, "cost": cost},
        headers={"Content-Type": "application/json"})
    return r.json()

def show_status():
    eid = get_endpoint_id()
    print(f"Endpoint: {ENDPOINT_NAME} (id={eid})")
    if not eid: print("  NOT DEPLOYED"); return
    workers = get_workers()
    print(f"  Workers: {len(workers)}")
    for w in workers:
        print(f"    id={w.get('id')} status={w.get('status')} load={w.get('cur_load',0):.1f}")
    route = get_route()
    print(f"  Route: {route.get('url', route)}")

def destroy():
    eid = get_endpoint_id()
    if not eid: print("Nothing to destroy"); return
    print(f"Destroying endpoint {eid}")
    requests.put(f"{VAST_API}/endptjobs/{eid}/", headers=headers(), json={"status": "stopped"})
    requests.delete(f"{VAST_API}/endptjobs/{eid}/", headers=headers())
    print("  Done")

def main():
    if not VAST_API_KEY: print("ERROR: VAST_KEY not set"); sys.exit(1)
    p = argparse.ArgumentParser()
    p.add_argument("action", choices=["create","status","workers","logs","route","destroy"])
    a = p.parse_args()
    if a.action == "create":
        eid = create_endpoint()
        if eid: create_workergroup(eid)
    elif a.action == "status": show_status()
    elif a.action == "workers": print(json.dumps(get_workers(), indent=2))
    elif a.action == "logs":
        eid = get_endpoint_id()
        if eid:
            ekey = get_endpoint_key()
            r = requests.post(f"{VAST_RUN}/get_endpoint_logs/",
                json={"id": eid, "api_key": ekey, "tail": 2000},
                headers={"Content-Type": "application/json"})
            for k,v in r.json().items():
                if v: print(f"\n=== {k} ===\n{v}")
    elif a.action == "route": print(json.dumps(get_route(), indent=2))
    elif a.action == "destroy": destroy()

if __name__ == "__main__": main()


Output:
Chunk ID: 05273d
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 3173
Output:
#!/usr/bin/env python3
"""
Production Stress Test for Hindi ASR - Simulates Real Customer Usage

Captures ALL metrics needed for scaling law decisions:
- Connection time (p50/p95/p99)
- First token latency (p50/p95/p99)
- End-to-end latency
- WER/CER (if reference text available)
- Throughput (streams * realtime)
- GPU utilization (via /metrics endpoint)
- Error rate
- Drift (latency accumulation over time)

Usage:
    python test_stress.py --url http://ip:8000 --mode ws --levels 1,10,50,100
    python test_stress.py --url http://ip:8000 --mode http --levels 1,5,10,20
    python test_stress.py --url http://ip:8000 --mode sse --levels 1,10,50
    python test_stress.py --endpoint hindi-asr-prod --mode http --levels 1,5,10
"""
import os, sys, json, time, math, struct, asyncio, argparse, csv, base64, wave
from pathlib import Path
import requests
from urllib.parse import urlparse
from datetime import datetime
import aiohttp

def generate_test_audio(duration_s=10.0, freq_hz=440, sample_rate=16000):
    """Generate test audio: sine wave as raw PCM + WAV"""
    n = int(sample_rate * duration_s)
    pcm = struct.pack(f'<{n}h', *[
        min(32767, max(-32768, int(16000 * math.sin(2 * math.pi * freq_hz * i / sample_rate))))
        for i in range(n)
    ])
    return pcm, pcm_to_wav_bytes(pcm, sample_rate), duration_s

def pcm_to_wav_bytes(pcm_bytes: bytes, sample_rate=16000) -> bytes:
    hdr = struct.pack(
        '<4sI4s4sIHHIIHH4sI',
        b'RIFF', 36 + len(pcm_bytes), b'WAVE', b'fmt ', 16,
        1, 1, sample_rate, sample_rate * 2, 2, 16,
        b'data', len(pcm_bytes),
    )
    return hdr + pcm_bytes

def load_wav_audio(path: str):
    with wave.open(path, "rb") as wf:
        channels = wf.getnchannels()
        sampwidth = wf.getsampwidth()
        rate = wf.getframerate()
        pcm = wf.readframes(wf.getnframes())
    if channels != 1:
        raise RuntimeError(f"{path}: expected mono WAV, got channels={channels}")
    if sampwidth != 2:
        raise RuntimeError(f"{path}: expected 16-bit WAV, got sampwidth={sampwidth}")
    if rate != 16000:
        raise RuntimeError(f"{path}: expected 16kHz WAV, got sample_rate={rate}")
    duration_s = len(pcm) / (16000.0 * 2.0)
    return pcm, pcm_to_wav_bytes(pcm, 16000), duration_s

def fit_audio_duration(pcm_bytes: bytes, target_duration_s: float):
    if target_duration_s <= 0:
        return pcm_bytes
    target_len = int(target_duration_s * 16000 * 2)
    if target_len <= 0:
        return pcm_bytes
    if len(pcm_bytes) == target_len:
        return pcm_bytes
    if len(pcm_bytes) > target_len:
        return pcm_bytes[:target_len]
    reps = math.ceil(target_len / len(pcm_bytes))
    out = (pcm_bytes * reps)[:target_len]
    return out

VAST_API = "https://console.vast.ai/api/v0"
VAST_RUN = "https://run.vast.ai"

def _env_or_dotenv(name: str):
    if os.environ.get(name):
        return os.environ.get(name)
    env_path = Path(__file__).resolve().parent.parent / ".env"
    if env_path.exists():
        for line in env_path.read_text().splitlines():
            if line.startswith(name + "="):
                return line.split("=", 1)[1].strip()
    return None

def get_endpoint_info(endpoint_name: str):
    account_key = _env_or_dotenv("VAST_KEY")
    if not account_key:
        raise RuntimeError("VAST_KEY not set; pass --endpoint-key")
    r = requests.get(
        f"{VAST_API}/endptjobs/",
        headers={"Accept": "application/json", "Authorization": f"Bearer {account_key}"},
        timeout=30,
    )
    data = r.json()
    results = data.get("results", []) if isinstance(data, dict) else data
    for ep in results or []:
        if ep.get("endpoint_name") == endpoint_name:
            return ep
    raise RuntimeError(f"endpoint '{endpoint_name}' not found")

def get_endpoint_key(endpoint_name: str, endpoint_key: str | None):
    if endpoint_key:
        return endpoint_key
    return get_endpoint_info(endpoint_name).get("api_key")

def get_endpoint_id(endpoint_name: str):
    return get_endpoint_info(endpoint_name).get("id")

def get_instance_info(instance_id: int):
    account_key = _env_or_dotenv("VAST_KEY")
    if not account_key:
        raise RuntimeError("VAST_KEY not set; required for instance port discovery")
    r = requests.get(
        f"{VAST_API}/instances/{instance_id}/",
        headers={"Accept": "application/json", "Authorization": f"Bearer {account_key}"},
        timeout=30,
    )
    d = r.json()
    if isinstance(d, dict):
        if "instances" in d and isinstance(d["instances"], dict):
            return d["instances"]
        if "results" in d and isinstance(d["results"], list) and d["results"]:
            return d["results"][0]
    raise RuntimeError(f"unexpected instance response for {instance_id}: {d}")

def get_endpoint_workers_sync(endpoint_name: str, endpoint_key: str):
    eid = get_endpoint_id(endpoint_name)
    r = requests.post(
        f"{VAST_RUN}/get_endpoint_workers/",
        json={"id": eid, "api_key": endpoint_key},
        timeout=30,
        headers={"Content-Type": "application/json"},
    )
    data = r.json()
    if isinstance(data, dict) and "workers" in data:
        return data["workers"]
    return data if isinstance(data, list) else []

def discover_ws_target(endpoint_name: str, endpoint_key: str, ws_port: int | None = None):
    workers = get_endpoint_workers_sync(endpoint_name, endpoint_key)
    active = [w for w in workers if str(w.get("status", "")).lower() in {"idle", "running"}]
    chosen = active[0] if active else (workers[0] if workers else None)
    if not chosen:
        raise RuntimeError("no endpoint workers available for WS discovery")

    instance_id = int(chosen["id"])
    inst = get_instance_info(instance_id)
    ip = inst.get("public_ipaddr")
    ports = inst.get("ports", {}) if isinstance(inst, dict) else {}
    host_port = ws_port
    if host_port is None:
        for key in ("18000/tcp", "18000"):
            if key in ports and ports[key]:
                host_port = int(ports[key][0].get("HostPort"))
                break
    if not ip or not host_port:
        raise RuntimeError(f"worker {instance_id} missing public ip or mapped ws port: ip={ip} ports={ports}")
    return {
        "ws_url": f"ws://{ip}:{host_port}/ws",
        "worker_id": instance_id,
        "host_port": int(host_port),
    }

def wrap_auth(route: dict, payload: dict):
    auth = {k: route[k] for k in ["signature", "cost", "endpoint", "reqnum", "url", "request_idx"] if k in route}
    return {"auth_data": auth, "payload": payload}

def pct(arr, p):
    if not arr: return 0
    s = sorted(arr)
    return s[min(int(len(s) * p / 100), len(s) - 1)]

class StressMetrics:
    """Collects and reports all metrics for a stress test level"""
    def __init__(self, concurrency, mode, audio_duration_s):
        self.concurrency = concurrency
        self.mode = mode
        self.audio_duration_s = audio_duration_s
        self.start_time = None
        self.end_time = None
        # Per-stream metrics
        self.connect_times_ms = []
        self.first_token_times_ms = []
        self.total_times_ms = []
        self.messages_per_stream = []
        self.errors = []
        self.texts = []
        self.drift_samples_ms = []
        self.drift_max_per_stream_ms = []
        # Server metrics snapshots
        self.server_metrics_before = None
        self.server_metrics_after = None

    def summary(self):
        ok = self.concurrency - len(self.errors)
        wall_time = (self.end_time - self.start_time) if self.end_time else 0
        throughput_x = 0.0
        if wall_time > 0:
            throughput_x = (ok * self.audio_duration_s) / wall_time
        return {
            "concurrency": self.concurrency,
            "mode": self.mode,
            "ok": ok,
            "errors": len(self.errors),
            "wall_time_s": round(wall_time, 2),
            "connect_ms_p50": round(pct(self.connect_times_ms, 50), 1),
            "connect_ms_p95": round(pct(self.connect_times_ms, 95), 1),
            "connect_ms_p99": round(pct(self.connect_times_ms, 99), 1),
            "first_token_ms_p50": round(pct(self.first_token_times_ms, 50), 1),
            "first_token_ms_p95": round(pct(self.first_token_times_ms, 95), 1),
            "first_token_ms_p99": round(pct(self.first_token_times_ms, 99), 1),
            "total_ms_p50": round(pct(self.total_times_ms, 50), 1),
            "total_ms_p95": round(pct(self.total_times_ms, 95), 1),
            "total_ms_p99": round(pct(self.total_times_ms, 99), 1),
            "drift_ms_p50": round(pct(self.drift_samples_ms, 50), 1),
            "drift_ms_p95": round(pct(self.drift_samples_ms, 95), 1),
            "drift_ms_p99": round(pct(self.drift_samples_ms, 99), 1),
            "drift_max_ms": round(pct(self.drift_max_per_stream_ms, 99), 1),
            "msgs_per_stream_avg": round(sum(self.messages_per_stream) / max(len(self.messages_per_stream), 1), 1),
            "throughput_x": round(throughput_x, 2),
        }

async def fetch_server_metrics(url=None, endpoint_name=None, endpoint_key=None, route_cost=10.0):
    """Fetch /metrics either directly or through serverless PyWorker."""
    if endpoint_name:
        try:
            async with aiohttp.ClientSession() as s:
                async with s.post(
                    f"{VAST_RUN}/route/",
                    json={"endpoint": endpoint_name, "api_key": endpoint_key, "cost": route_cost},
                    timeout=aiohttp.ClientTimeout(total=10),
                ) as r:
                    route = await r.json()
            if "url" not in route:
                return {}
            worker_url = route["url"].rstrip("/")
            req = wrap_auth(route, {})
            async with aiohttp.ClientSession() as s:
                async with s.post(
                    f"{worker_url}/metrics",
                    json=req,
                    timeout=aiohttp.ClientTimeout(total=10),
                ) as r:
                    if r.status == 200:
                        return await r.json()
        except Exception:
            return {}
        return {}

    if not url:
        return {}
    try:
        async with aiohttp.ClientSession() as s:
            async with s.get(f"{url}/metrics", timeout=aiohttp.ClientTimeout(total=5)) as r:
                if r.status == 200:
                    return await r.json()
    except Exception:
        pass
    return {}

async def fetch_endpoint_workers(endpoint_name, endpoint_key):
    """Fetch worker list for endpoint from Vast run API"""
    try:
        eid = get_endpoint_id(endpoint_name)
        async with aiohttp.ClientSession() as s:
            async with s.post(
                f"{VAST_RUN}/get_endpoint_workers/",
                json={"id": eid, "api_key": endpoint_key},
                timeout=aiohttp.ClientTimeout(total=10),
            ) as r:
                data = await r.json()
                if isinstance(data, dict) and "workers" in data:
                    return data["workers"]
                return data if isinstance(data, list) else []
    except Exception:
        return []

async def run_ws_stream(ws_url, pcm_bytes, stream_idx, metrics, duration_s, chunk_duration_ms=160):
    """Run a single WebSocket stream at real-time pace"""
    chunk_size = int((chunk_duration_ms / 1000.0) * 16000 * 2)
    t_start = time.perf_counter()

    try:
        async with aiohttp.ClientSession() as s:
            async with s.ws_connect(ws_url, timeout=45, heartbeat=30) as ws:
                t_connected = time.perf_counter()
                metrics.connect_times_ms.append((t_connected - t_start) * 1000)

                # Wait for ready
                ready = await asyncio.wait_for(ws.receive(), timeout=10)
                if ready.type != aiohttp.WSMsgType.TEXT:
                    metrics.errors.append({"stream": stream_idx, "error": f"ready message type={ready.type}"})
                    return

                first_token = None
                msgs = []
                got_end = False
                stream_drifts_ms = []

                async def send():
                    for i in range(0, len(pcm_bytes), chunk_size):
                        if ws.closed:
                            break
                        try:
                            await ws.send_bytes(pcm_bytes[i:i + chunk_size])
                        except Exception:
                            break
                        await asyncio.sleep(chunk_duration_ms / 1000.0)
                    if not ws.closed:
                        try:
                            await ws.send_str("END")
                        except Exception:


Output:
Chunk ID: 20b690
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 2359
Output:
#!/usr/bin/env python3
"""
Full-fidelity stress + quality test for Hindi ASR over Vast serverless (HTTP/SSE).

Features:
- Customer-like concurrent clients with multiple requests per client
- Route timing + queue-full detection
- First-token/total latency + server processing time
- Optional FLEURS Hindi audio (real speech) with WER/CER checks
- Time-series snapshots from /metrics and /get_endpoint_workers/
- CSV + JSON outputs suitable for scaling-law fitting
"""

import argparse
import asyncio
import base64
import csv
import io
import json
import math
import os
import random
import re
import struct
import sys
import time
import wave
from collections import Counter
from dataclasses import dataclass, asdict
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple

import aiohttp
import numpy as np
import requests

try:
    import jiwer
except Exception:
    jiwer = None

VAST_API = "https://console.vast.ai/api/v0"
VAST_RUN = "https://run.vast.ai"

def _env_or_dotenv(name: str) -> Optional[str]:
    if os.environ.get(name):
        return os.environ.get(name)
    env_path = Path(__file__).resolve().parent.parent / ".env"
    if env_path.exists():
        for line in env_path.read_text().splitlines():
            if line.startswith(name + "="):
                return line.split("=", 1)[1].strip()
    return None

def get_endpoint_info(endpoint_name: str, account_key: Optional[str] = None) -> Dict[str, Any]:
    key = account_key or _env_or_dotenv("VAST_KEY")
    if not key:
        raise RuntimeError("VAST_KEY not set; pass --endpoint-key and --endpoint-id")
    r = requests.get(
        f"{VAST_API}/endptjobs/",
        headers={"Accept": "application/json", "Authorization": f"Bearer {key}"},
        timeout=30,
    )
    data = r.json()
    results = data.get("results", []) if isinstance(data, dict) else data
    for ep in results or []:
        if ep.get("endpoint_name") == endpoint_name:
            return ep
    raise RuntimeError(f"endpoint '{endpoint_name}' not found")

async def get_route(session: aiohttp.ClientSession, endpoint_name: str, endpoint_key: str, cost: float) -> Tuple[Optional[Dict[str, Any]], float, Optional[str], bool]:
    t0 = time.perf_counter()
    try:
        async with session.post(
            f"{VAST_RUN}/route/",
            json={"endpoint": endpoint_name, "api_key": endpoint_key, "cost": cost},
            timeout=aiohttp.ClientTimeout(total=15),
        ) as r:
            data = await r.json(content_type=None)
    except Exception as e:
        return None, (time.perf_counter() - t0) * 1000.0, str(e), False

    dt = (time.perf_counter() - t0) * 1000.0
    if isinstance(data, dict) and "url" in data:
        return data, dt, None, False

    status = ""
    if isinstance(data, dict):
        status = str(data.get("status", ""))
    queue_full = "all worker queues full" in status.lower()
    return None, dt, f"route failed: {data}", queue_full

def wrap_auth(route: Dict[str, Any], payload: Dict[str, Any]) -> Dict[str, Any]:
    auth = {k: route[k] for k in ["signature", "cost", "endpoint", "reqnum", "url", "request_idx"] if k in route}
    return {"auth_data": auth, "payload": payload}

def pct(vals: List[float], p: int) -> float:
    if not vals:
        return 0.0
    s = sorted(vals)
    idx = min(int(len(s) * p / 100), len(s) - 1)
    return float(s[idx])

def safe_mean(vals: List[float]) -> float:
    return float(sum(vals) / len(vals)) if vals else 0.0

def normalize_text(text: str) -> str:
    if not text:
        return ""
    text = text.lower().strip()
    text = re.sub(r"<[^>]+>", " ", text)
    text = re.sub(r"[^\u0900-\u097fa-z0-9\s]", " ", text)
    text = re.sub(r"\s+", " ", text).strip()
    return text

def compute_wer_cer(reference: str, hypothesis: str) -> Tuple[Optional[float], Optional[float]]:
    if not jiwer:
        return None, None
    ref = normalize_text(reference)
    hyp = normalize_text(hypothesis)
    if not ref:
        return None, None
    try:
        return float(jiwer.wer(ref, hyp)), float(jiwer.cer(ref, hyp))
    except Exception:
        return None, None

@dataclass
class AudioSample:
    sample_id: str
    duration_s: float
    wav_bytes: bytes
    reference: Optional[str] = None

def _pcm_to_wav_bytes(pcm_int16: np.ndarray, sample_rate: int = 16000) -> bytes:
    bio = io.BytesIO()
    with wave.open(bio, "wb") as wf:
        wf.setnchannels(1)
        wf.setsampwidth(2)
        wf.setframerate(sample_rate)
        wf.writeframes(pcm_int16.astype(np.int16).tobytes())
    return bio.getvalue()

def make_sine_samples(durations: List[float]) -> List[AudioSample]:
    samples: List[AudioSample] = []
    for i, d in enumerate(durations):
        n = int(16000 * d)
        pcm = np.array(
            [
                min(32767, max(-32768, int(16000 * math.sin(2 * math.pi * 440 * k / 16000))))
                for k in range(n)
            ],
            dtype=np.int16,
        )
        wav = _pcm_to_wav_bytes(pcm)
        samples.append(AudioSample(sample_id=f"sine_{i}_{d:.2f}s", duration_s=float(d), wav_bytes=wav, reference=None))
    return samples

def load_fleurs_samples(max_samples: int, min_duration: float, max_duration: float, seed: int) -> List[AudioSample]:
    from datasets import load_dataset

    ds = load_dataset("google/fleurs", "hi_in", split="test")
    rows = list(ds)
    rnd = random.Random(seed)
    rnd.shuffle(rows)

    out: List[AudioSample] = []
    for row in rows:
        audio = row["audio"]
        arr = np.asarray(audio["array"])  # float32 [-1, 1]
        sr = int(audio["sampling_rate"])
        if sr != 16000:
            # FLEURS hi_in is normally 16kHz, but keep explicit guard.
            continue
        duration = len(arr) / float(sr)
        if duration < min_duration or duration > max_duration:
            continue
        pcm = np.clip(arr, -1.0, 1.0)
        pcm = (pcm * 32767.0).astype(np.int16)
        wav = _pcm_to_wav_bytes(pcm, sample_rate=sr)
        out.append(
            AudioSample(
                sample_id=str(row["id"]),
                duration_s=round(duration, 3),
                wav_bytes=wav,
                reference=row.get("transcription"),
            )
        )
        if len(out) >= max_samples:
            break

    if not out:
        raise RuntimeError("No FLEURS samples matched filters")
    return out

@dataclass
class RequestRecord:
    client_idx: int
    req_idx: int
    mode: str
    sample_id: str
    duration_s: float
    route_ms: float
    connect_ms: float
    first_token_ms: float
    total_ms: float
    server_processing_ms: float
    status_code: int
    ok: bool
    queue_full: bool
    error: str
    reference: str
    hypothesis: str
    wer: float
    cer: float

@dataclass
class LevelResult:
    mode: str
    concurrency: int
    requests_per_client: int
    wall_time_s: float
    ok: int
    errors: int
    queue_full_errors: int
    throughput_x: float
    route_ms_p50: float
    route_ms_p95: float
    route_ms_p99: float
    first_token_ms_p50: float
    first_token_ms_p95: float
    first_token_ms_p99: float
    total_ms_p50: float
    total_ms_p95: float
    total_ms_p99: float
    server_proc_ms_p50: float
    server_proc_ms_p95: float
    server_proc_ms_p99: float
    vram_used_gb_max: float
    worker_cur_load_max: float
    worker_cur_load_p95: float
    worker_cur_perf_avg: float
    wer_pct: float
    cer_pct: float
    status_counts: Dict[str, int]

async def fetch_server_metrics_via_endpoint(
    session: aiohttp.ClientSession,
    endpoint_name: str,
    endpoint_key: str,
    route_cost: float,
) -> Dict[str, Any]:
    route, _, err, _ = await get_route(session, endpoint_name, endpoint_key, route_cost)
    if not route:
        return {"error": err}
    worker_url = route["url"].rstrip("/")
    req = wrap_auth(route, {})
    try:
        async with session.post(
            f"{worker_url}/metrics",
            json=req,
            timeout=aiohttp.ClientTimeout(total=12),
        ) as r:
            if r.status == 200:
                return await r.json(content_type=None)
            return {"error": f"metrics status={r.status}"}
    except Exception as e:
        return {"error": str(e)}

async def fetch_endpoint_workers(
    session: aiohttp.ClientSession,
    endpoint_id: int,
    endpoint_key: str,
) -> List[Dict[str, Any]]:
    try:
        async with session.post(
            f"{VAST_RUN}/get_endpoint_workers/",
            json={"id": endpoint_id, "api_key": endpoint_key},
            timeout=aiohttp.ClientTimeout(total=12),
        ) as r:
            data = await r.json(content_type=None)
            if isinstance(data, dict) and "workers" in data:
                return data["workers"]
            return data if isinstance(data, list) else []
    except Exception:
        return []

async def metrics_sampler(
    stop_evt: asyncio.Event,
    interval_s: float,
    session: aiohttp.ClientSession,
    endpoint_name: Optional[str],
    endpoint_key: Optional[str],
    endpoint_id: Optional[int],
    route_cost: float,
    server_samples: List[Dict[str, Any]],
    worker_samples: List[Dict[str, Any]],
):
    while not stop_evt.is_set():
        ts = time.time()
        if endpoint_name and endpoint_key:
            sm = await fetch_server_metrics_via_endpoint(session, endpoint_name, endpoint_key, route_cost)


Summary: auto


**Searching for benchmark scripts**


I found prior benchmark artifacts and existing full-stress harnesses, so I’m now locating all H100/WS-related knobs in code and docs to run an apples-to-apples serverless benchmark and isolate the bottlenecks.


rg -n "H100|h100|MAX_ACTIVE_STREAMS|att_context_size|chunk|ws|websocket|route_cost|queue full|worker queues full" production vast_docs | sed -n '1,260p'


Output:
Chunk ID: 243d40
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 4417
Output:
vast_docs/pyworker.md:45:The [Vast PyWorker repository](https://github.com/vast-ai/pyworker/) gives examples that are useful for learning how to create a custom PyWorker for your custom template and integrate with Vast’s Serverless system. Even with a custom PyWorker, the PyWorker code runs on your Vast instance, and we automate its installation and activation during instance creation. The graphic below shows how the files and entities for the Serverless system are organized.
vast_docs/custom_pyworkers.md:124:   * Support both standard JSON responses and streaming (SSE, NDJSON, chunked) responses.
vast_docs/custom_pyworkers.md:254:  * On the first match of any `on_load` prefix, the worker knows the model is “loaded” and can begin benchmarking.
vast_docs/custom_pyworkers.md:455:  * `Transfer-Encoding: chunked`
vast_docs/custom_pyworkers.md:460:  * Streams chunks from the model server to the client as they arrive.
vast_docs/serverless_params.md:101:A command-line style string containing additional parameters for instance creation that will be parsed and applied when the serverless engine creates new workers. This allows you to customize instance configuration beyond what's specified in templates.
vast_docs/inside_serverless_gpu.md:37:On Vast Serverless, the only way to access the ML model is through the PyWorker that wraps it. This allows the PyWorker to report accurate metrics to the serverless system so it can size the number of GPU instances appropriatley.
vast_docs/creating_endpoints_workergroups.md:131:* `launch_args` (string): A command-line style string containing additional parameters for instance creation that will be parsed and applied when the autoscaler creates new workers. This allows you to customize instance configuration beyond what's specified in templates.
production/test_smoke.py:97:def discover_ws_url(endpoint_name: str, endpoint_key: str):
production/test_smoke.py:114:    return f"ws://{ip}:{host_port}/ws", instance_id
production/test_smoke.py:141:async def test_http_batch(url, endpoint_name=None, endpoint_key=None, route_cost=10.0):
production/test_smoke.py:149:                route = await asyncio.to_thread(get_route, endpoint_name, endpoint_key, route_cost)
production/test_smoke.py:161:                        route = await asyncio.to_thread(get_route, endpoint_name, endpoint_key, route_cost)
production/test_smoke.py:194:async def test_http_stream(url, endpoint_name=None, endpoint_key=None, route_cost=10.0):
production/test_smoke.py:202:                route = await asyncio.to_thread(get_route, endpoint_name, endpoint_key, route_cost)
production/test_smoke.py:204:                payload = {"audio_base64": audio_b64, "language": "hi", "chunk_duration_ms": 160}
production/test_smoke.py:248:async def test_websocket(url, endpoint_name=None, endpoint_key=None, route_cost=10.0, ws_port=None):
production/test_smoke.py:252:            if ws_port:
production/test_smoke.py:253:                route = await asyncio.to_thread(get_route, endpoint_name, endpoint_key, route_cost)
production/test_smoke.py:255:                scheme = "wss" if parsed.scheme == "https" else "ws"
production/test_smoke.py:257:                ws_url = f"{scheme}://{host}:{ws_port}/ws"
production/test_smoke.py:258:                print(f"  Using explicit --ws-port={ws_port}: {ws_url}")
production/test_smoke.py:260:                ws_url, worker_id = await asyncio.to_thread(discover_ws_url, endpoint_name, endpoint_key)
production/test_smoke.py:261:                print(f"  Discovered worker={worker_id} ws_url={ws_url}")
production/test_smoke.py:266:        ws_url = url.replace("http://","ws://").replace("https://","wss://") + "/ws"
production/test_smoke.py:271:            async with s.ws_connect(ws_url, timeout=30) as ws:
production/test_smoke.py:274:                ready = await asyncio.wait_for(ws.receive(), timeout=10)
production/test_smoke.py:278:                    print(f"  [FAIL] closed before ready: code={ws.close_code}")
production/test_smoke.py:280:                chunk_size = int(0.160*16000*2)
production/test_smoke.py:284:                    for i in range(0, len(pcm), chunk_size):
production/test_smoke.py:285:                        if ws.closed:
production/test_smoke.py:288:                            await ws.send_bytes(pcm[i:i+chunk_size])
production/test_smoke.py:292:                    if not ws.closed:
production/test_smoke.py:294:                            await ws.send_str("END")
production/test_smoke.py:300:                        m = await asyncio.wait_for(ws.receive(), timeout=15)
production/test_smoke.py:316:                print(f"  [{'PASS' if ok else 'FAIL'}] {len(msgs)} msgs {dt:.0f}ms first_token={ft_ms:.0f}ms close_code={ws.close_code}")
production/test_smoke.py:330:    p.add_argument("--ws-port", type=int, default=None, help="Direct WS port on worker (if exposed)")
production/test_smoke.py:332:    p.add_argument("--ws-only", action="store_true")
production/test_smoke.py:340:    if not a.ws_only:
production/test_smoke.py:341:        results["http_batch"] = await test_http_batch(url, a.endpoint, endpoint_key, a.route_cost)
production/test_smoke.py:342:        results["http_stream"] = await test_http_stream(url, a.endpoint, endpoint_key, a.route_cost)
production/test_smoke.py:344:        results["websocket"] = await test_websocket(url, a.endpoint, endpoint_key, a.route_cost, a.ws_port)
production/onstart.sh:8:export MAX_ACTIVE_STREAMS="${MAX_ACTIVE_STREAMS:-100}"
production/test_stress_full.py:94:    queue_full = "all worker queues full" in status.lower()
production/test_stress_full.py:176:    rows = list(ds)
production/test_stress_full.py:178:    rnd.shuffle(rows)
production/test_stress_full.py:181:    for row in rows:
production/test_stress_full.py:267:    route_cost: float,
production/test_stress_full.py:269:    route, _, err, _ = await get_route(session, endpoint_name, endpoint_key, route_cost)
production/test_stress_full.py:313:    route_cost: float,
production/test_stress_full.py:320:            sm = await fetch_server_metrics_via_endpoint(session, endpoint_name, endpoint_key, route_cost)
production/test_stress_full.py:333:    route_cost: float,
production/test_stress_full.py:348:            route, route_ms, rerr, qf = await get_route(session, endpoint_name, endpoint_key, route_cost)
production/test_stress_full.py:368:                    route2, route_ms2, rerr2, qf2 = await get_route(session, endpoint_name, endpoint_key, route_cost)
production/test_stress_full.py:447:    route_cost: float,
production/test_stress_full.py:461:            route, route_ms, rerr, qf = await get_route(session, endpoint_name, endpoint_key, route_cost)
production/test_stress_full.py:469:                "chunk_duration_ms": 160,
production/test_stress_full.py:680:    route_cost: float,
production/test_stress_full.py:724:                route_cost,
production/test_stress_full.py:741:                        route_cost,
production/test_stress_full.py:750:                        route_cost,
production/test_stress_full.py:958:                route_cost=args.route_cost,
production/test_stress.py:16:    python test_stress.py --url http://ip:8000 --mode ws --levels 1,10,50,100
production/test_stress.py:143:def discover_ws_target(endpoint_name: str, endpoint_key: str, ws_port: int | None = None):
production/test_stress.py:154:    host_port = ws_port
production/test_stress.py:161:        raise RuntimeError(f"worker {instance_id} missing public ip or mapped ws port: ip={ip} ports={ports}")
production/test_stress.py:163:        "ws_url": f"ws://{ip}:{host_port}/ws",
production/test_stress.py:230:async def fetch_server_metrics(url=None, endpoint_name=None, endpoint_key=None, route_cost=10.0):
production/test_stress.py:237:                    json={"endpoint": endpoint_name, "api_key": endpoint_key, "cost": route_cost},
production/test_stress.py:286:async def run_ws_stream(ws_url, pcm_bytes, stream_idx, metrics, duration_s, chunk_duration_ms=160):
production/test_stress.py:288:    chunk_size = int((chunk_duration_ms / 1000.0) * 16000 * 2)
production/test_stress.py:293:            async with s.ws_connect(ws_url, timeout=45, heartbeat=30) as ws:
production/test_stress.py:298:                ready = await asyncio.wait_for(ws.receive(), timeout=10)
production/test_stress.py:309:                    for i in range(0, len(pcm_bytes), chunk_size):
production/test_stress.py:310:                        if ws.closed:
production/test_stress.py:313:                            await ws.send_bytes(pcm_bytes[i:i + chunk_size])
production/test_stress.py:316:                        await asyncio.sleep(chunk_duration_ms / 1000.0)
production/test_stress.py:317:                    if not ws.closed:
production/test_stress.py:319:                            await ws.send_str("END")
production/test_stress.py:327:                            m = await asyncio.wait_for(ws.receive(), timeout=30)
production/test_stress.py:368:                    metrics.errors.append({"stream": stream_idx, "error": f"ws closed before output (close_code={ws.close_code})"})
production/test_stress.py:375:                         endpoint_name=None, endpoint_key=None, route_cost=10.0):
production/test_stress.py:385:                    json={"endpoint": endpoint_name, "api_key": endpoint_key, "cost": route_cost},
production/test_stress.py:412:                            json={"endpoint": endpoint_name, "api_key": endpoint_key, "cost": route_cost},
production/test_stress.py:463:                         endpoint_name=None, endpoint_key=None, route_cost=10.0):
production/test_stress.py:472:                    json={"endpoint": endpoint_name, "api_key": endpoint_key, "cost": route_cost},
production/test_stress.py:480:                payload = {"audio_base64": audio_b64, "language": "hi", "chunk_duration_ms": 160}
production/test_stress.py:539:                           endpoint_name=None, endpoint_key=None, route_cost=10.0,
production/test_stress.py:540:                           ws_url=None, chunk_duration_ms=160,
production/test_stress.py:559:            route_cost=route_cost,
production/test_stress.py:569:        if mode == "ws":
production/test_stress.py:570:            tasks.append(run_ws_stream(ws_url, pcm, i, metrics, audio_duration_s, chunk_duration_ms))
production/test_stress.py:572:            tasks.append(run_http_batch(url, wav, i, metrics, endpoint_name, endpoint_key, route_cost))
production/test_stress.py:574:            tasks.append(run_sse_stream(url, wav, i, metrics, endpoint_name, endpoint_key, route_cost))
production/test_stress.py:585:            route_cost=route_cost,
production/test_stress.py:598:    if mode == "ws":
production/test_stress.py:632:    p.add_argument("--ws-port", type=int, default=None, help="Host WS port override for endpoint mode")
production/test_stress.py:633:    p.add_argument("--chunk-ms", type=int, default=160, help="WS client chunk duration in ms")
production/test_stress.py:634:    p.add_argument("--mode", default="ws", choices=["ws", "http", "sse", "both"])
production/test_stress.py:647:    modes = [a.mode] if a.mode != "both" else ["ws", "http", "sse"]
production/test_stress.py:650:    resolved_ws_url = None
production/test_stress.py:662:    if "ws" in modes:
production/test_stress.py:663:        print(f"WS chunk: {a.chunk_ms}ms")
production/test_stress.py:683:        if "ws" in modes:
production/test_stress.py:685:                target = await asyncio.to_thread(discover_ws_target, a.endpoint, endpoint_key, a.ws_port)
production/test_stress.py:686:                resolved_ws_url = target["ws_url"]
production/test_stress.py:687:                print(f"WS target: worker={target['worker_id']} url={resolved_ws_url}")
production/test_stress.py:689:                print(f"Cannot resolve websocket target: {e}")
production/test_stress.py:692:        if "ws" in modes:
production/test_stress.py:693:            resolved_ws_url = url.replace("http://", "ws://").replace("https://", "wss://") + "/ws"
production/test_stress.py:694:            print(f"WS target: {resolved_ws_url}")
production/test_stress.py:720:                a.route_cost,
production/test_stress.py:721:                resolved_ws_url,
production/test_stress.py:722:                a.chunk_ms,
production/server.py:40:from starlette.websockets import WebSocketState
production/server.py:64:MAX_ACTIVE_STREAMS = int(os.environ.get("MAX_ACTIVE_STREAMS", "100"))
production/server.py:85:    att_context_size: list = field(default_factory=lambda: [70, 6])
production/server.py:129:            "vad_filtered_chunks": 0,
production/server.py:133:            "ws_connections_total": 0,
production/server.py:134:            "ws_connections_active": 0,
production/server.py:135:            "ws_connections_rejected": 0,
production/server.py:244:        att_context_size: List[int] = field(default_factory=lambda: [70, 6])
production/server.py:247:        chunk_size_in_secs: Optional[float] = None
production/server.py:310:    logger.info(f"  att_context_size={state.cfg.streaming.att_context_size}")
production/server.py:334:    chunk_size = int(0.080 * 16000 * 2)  # 80ms chunks, 16kHz, 16-bit
production/server.py:341:        for i in range(0, len(audio_bytes), chunk_size):
production/server.py:342:            chunk = audio_bytes[i:i + chunk_size]
production/server.py:343:            state.pipeline.append_streaming_audio(stream_id, chunk)
production/server.py:387:    # Use pipeline's native chunk size if available; fallback to 80ms
production/server.py:388:    frame_secs = getattr(state.pipeline, "chunk_size_in_secs", 0.08) or 0.08
production/server.py:395:        # Feed audio in frame-sized chunks
production/server.py:397:            chunk = pcm_bytes[i:i + frame_bytes]
production/server.py:398:            state.pipeline.append_streaming_audio(stream_id, chunk)
production/server.py:459:    Returns (audio_bytes, language, response_format, chunk_duration_ms).
production/server.py:478:        chunk_duration_ms = int(body.get("chunk_duration_ms", 160) or 160)
production/server.py:479:        return audio_data, language, response_format, chunk_duration_ms
production/server.py:494:    chunk_duration_ms = int(form.get("chunk_duration_ms", 160) or 160)
production/server.py:495:    return audio_data, language, response_format, chunk_duration_ms
production/server.py:518:    status = "healthy" if active < MAX_ACTIVE_STREAMS else "overloaded"
production/server.py:527:            "max_streams": MAX_ACTIVE_STREAMS,
production/server.py:565:        "max_streams": MAX_ACTIVE_STREAMS,
production/server.py:570:            "vad_filtered_chunks": m["vad_filtered_chunks"],
production/server.py:582:        "ws": {
production/server.py:583:            "connections_total": m["ws_connections_total"],
production/server.py:584:            "connections_active": m["ws_connections_active"],
production/server.py:585:            "connections_rejected": m["ws_connections_rejected"],
production/server.py:625:        audio_data, language, _response_format, _chunk_duration_ms = await _parse_audio_request(request)
production/server.py:665:        audio_data, language, response_format, _chunk_duration_ms = await _parse_audio_request(request)
production/server.py:715:        audio_data, language, response_format, chunk_duration_ms = await _parse_audio_request(request)
production/server.py:730:            chunk_size = int(chunk_duration_ms / 1000.0 * 16000 * 2)  # bytes per chunk
production/server.py:731:            att_ctx = state.cfg.streaming.att_context_size[1] + 1
production/server.py:739:                for i in range(0, len(pcm_bytes), chunk_size):
production/server.py:740:                    chunk = pcm_bytes[i:i + chunk_size]
production/server.py:741:                    audio_buffer += chunk
production/server.py:830:@app.websocket("/ws")
production/server.py:831:async def websocket_streaming(ws: WebSocket):
production/server.py:836:      Client -> Server: binary audio chunks (raw PCM 16-bit 16kHz mono)
production/server.py:843:    state.metrics["ws_connections_total"] += 1
production/server.py:847:    if active >= MAX_ACTIVE_STREAMS:
production/server.py:848:        state.metrics["ws_connections_rejected"] += 1
production/server.py:849:        await ws.accept()
production/server.py:850:        await ws.close(code=1013, reason=f"Overloaded: {active}/{MAX_ACTIVE_STREAMS}")
production/server.py:853:    await ws.accept()
production/server.py:854:    state.metrics["ws_connections_active"] += 1
production/server.py:872:        state.metrics["ws_connections_active"] -= 1
production/server.py:875:        if ws.application_state is WebSocketState.CONNECTED:
production/server.py:876:            await ws.close()
production/server.py:880:    await ws.send_text(json.dumps({"type": "Ready", "id": stream_id}))
production/server.py:884:        """Receive audio chunks from client, feed to pipeline"""
production/server.py:885:        att_ctx = state.cfg.streaming.att_context_size[1] + 1
production/server.py:891:                message = await ws.receive()
production/server.py:910:                            state.metrics["vad_filtered_chunks"] += 1
production/server.py:967:                            await ws.send_text("END")
production/server.py:976:                    await ws.send_text(json_str)
production/server.py:992:        state.metrics["ws_connections_active"] -= 1
production/server.py:994:        if ws.application_state is WebSocketState.CONNECTED:
production/server.py:995:            await ws.close()
production/server.py:1221:        f"vad_skip={m['vad_filtered_chunks']}"
production/server.py:1242:    logger.info(f"  Max streams: {MAX_ACTIVE_STREAMS}")


Output:
Chunk ID: 3fef2e
Wall time: 0.0516 seconds
Process exited with code 0
Original token count: 2107
Output:
"""
Production Hindi ASR Server - Standalone FastAPI
Supports: HTTP batch, HTTP streaming (SSE), WebSocket streaming
Designed for Vast.ai serverless deployment with PyWorker integration.

Architecture:
  - NeMo cache-aware RNNT streaming pipeline (Nemotron-0.6B fine-tuned Hindi)
  - Event-driven batching (50ms max wait, immediate dispatch when batch fills)
  - CPU-based Silero VAD gating (filters silence before GPU, ~3x effective capacity)
  - Per-stream state management with fairness scheduling
  - Backpressure: rejects new connections when at capacity (fast-fail > silent degrade)

Performance (A100-SXM4-80GB, from prior benchmarks):
  - Safe zone: 1-50 concurrent streams (WER 22-31%, FT <211ms)
  - Optimal: 50-100 streams (WER 31-33%, FT <402ms)
  - Edge: 100-140 streams (WER 33-37%)
  - VRAM: 10.4GB constant (compute-bound, not memory-bound)
"""

import os
import sys
import time
import json
import wave
import asyncio
import logging
import tempfile
import signal
import base64
from io import BytesIO
from dataclasses import dataclass, field
from typing import Optional, List
from collections import deque

import numpy as np
import torch
import uvicorn
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
from starlette.websockets import WebSocketState

# === Logging setup ===
LOG_FILE = os.environ.get("MODEL_LOG", "/var/log/model/server.log")
os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[
        logging.FileHandler(LOG_FILE),
        logging.StreamHandler(sys.stdout),
    ],
)
logger = logging.getLogger("asr_server")

# === Configuration ===

MODEL_PATH = os.environ.get("MODEL_PATH", "/model/final_model.nemo")
HF_MODEL_REPO = os.environ.get("HF_MODEL_REPO", "BayAreaBoys/nemotron-hindi")
HF_MODEL_FILE = os.environ.get("HF_MODEL_FILE", "final_model.nemo")
SERVER_PORT = int(os.environ.get("SERVER_PORT", "8000"))

# Production limits
MAX_ACTIVE_STREAMS = int(os.environ.get("MAX_ACTIVE_STREAMS", "100"))
MAX_BATCH_WAIT_MS = float(os.environ.get("MAX_BATCH_WAIT_MS", "50"))
ENABLE_VAD = os.environ.get("ENABLE_VAD", "true").lower() == "true"
VAD_THRESHOLD = float(os.environ.get("VAD_THRESHOLD", "0.5"))

@dataclass
class PipelineConfig:
    """Pipeline configuration — matches NeMo's expected OmegaConf structure"""
    model_name: str = ""
    device: str = "cuda"
    device_id: int = 0
    compute_dtype: str = "bfloat16"
    use_amp: bool = True
    # Decoding
    decoding_strategy: str = "greedy_batch"
    use_cuda_graph_decoder: bool = False
    max_symbols: int = 10
    # Streaming
    sample_rate: int = 16000
    batch_size: int = 512
    att_context_size: list = field(default_factory=lambda: [70, 6])
    use_cache: bool = True
    use_feat_cache: bool = True
    num_slots: int = 1024
    # Endpointing
    stop_history_eou: int = 800
    residue_tokens_at_end: int = 2

# === Global server state ===
class ASRServerState:
    """
    Centralized server state. Single instance per process.
    WHY: Avoids global variable soup, enables clean shutdown.
    """
    def __init__(self):
        self.pipeline = None
        self.cfg = None
        self.vad = None
        self.start_time = None

        # Stream management
        self.client_queues: dict[int, asyncio.Queue] = {}
        self.client_queues_lock = asyncio.Lock()
        self.stream_audio_timestamps: dict[int, float] = {}
        self.stream_timestamps_lock = asyncio.Lock()
        self.streams_pending_cleanup: set[int] = set()
        self.streams_cleanup_lock = asyncio.Lock()

        # Event-driven batching
        self._batch_ready_event = asyncio.Event()
        self.inference_task = None
        self.inference_running = False

        # Metrics (rolling window for percentile calculation)
        self.metrics = {
            "total_batches": 0,
            "total_outputs": 0,
            "total_gpu_ms": 0.0,
            "total_route_ms": 0.0,
            "batch_sizes": deque(maxlen=500),
            "gpu_times_ms": deque(maxlen=500),
            "cycle_times_ms": deque(maxlen=500),
            "empty_cycles": 0,
            "vad_filtered_chunks": 0,
            "http_requests_total": 0,
            "http_requests_ok": 0,
            "http_requests_err": 0,
            "ws_connections_total": 0,
            "ws_connections_active": 0,
            "ws_connections_rejected": 0,
        }
        self.model_loaded = False

state = ASRServerState()

# === FastAPI App ===
app = FastAPI(
    title="Hindi ASR Production Server",
    description="Cache-aware streaming ASR for Hindi (Nemotron-0.6B fine-tuned)",
    version="1.0.0",
)

# ============================================================
# Model Loading
# ============================================================

def download_model_if_needed() -> str:
    """Download model from HuggingFace if not found locally. Returns local path."""
    if os.path.exists(MODEL_PATH):
        logger.info(f"Model found at {MODEL_PATH}")
        return MODEL_PATH

    logger.info(f"Model not found at {MODEL_PATH}, downloading from {HF_MODEL_REPO}...")
    from huggingface_hub import hf_hub_download
    hf_token = os.environ.get("HF_TOKEN")
    model_path = hf_hub_download(
        repo_id=HF_MODEL_REPO,
        filename=HF_MODEL_FILE,
        token=hf_token,
        cache_dir="/model",
    )
    logger.info(f"Model downloaded to {model_path}")
    return model_path

def build_pipeline_config(model_path: str):
    """Build OmegaConf config for NeMo pipeline"""
    from omegaconf import OmegaConf
    from dataclasses import dataclass, field
    from typing import Optional, List

    # Full config structure matching CacheAwarePipelineConfig from nemotron_asr
    @dataclass
    class BoostingTreeConfig:
        model_path: Optional[str] = None
        key_phrases_file: Optional[str] = None
        key_phrases_list: Optional[List[str]] = None
        source_lang: str = "en"

    @dataclass
    class GreedyDecodingConfig:
        use_cuda_graph_decoder: bool = False
        max_symbols: int = 10
        ngram_lm_model: Optional[str] = None
        ngram_lm_alpha: float = 0.0
        boosting_tree: BoostingTreeConfig = field(default_factory=BoostingTreeConfig)
        boosting_tree_alpha: float = 0.0

    @dataclass
    class DecodingConfig:
        strategy: str = "greedy_batch"
        preserve_alignments: bool = False
        fused_batch_size: int = -1
        greedy: GreedyDecodingConfig = field(default_factory=GreedyDecodingConfig)

    @dataclass
    class ASRConfig:
        model_name: str = model_path
        device: str = "cuda"
        device_id: int = 0
        compute_dtype: str = "bfloat16"
        use_amp: bool = True
        decoding: DecodingConfig = field(default_factory=DecodingConfig)

    @dataclass
    class ITNConfig:
        input_case: str = "lower_cased"
        whitelist: Optional[str] = None
        overwrite_cache: bool = False
        max_number_of_permutations_per_split: int = 729
        left_padding_size: int = 4
        batch_size: int = 32
        n_jobs: int = 16

    @dataclass
    class ConfidenceConfig:
        exclude_blank: bool = True
        aggregation: str = "mean"
        method_cfg: dict = field(default_factory=lambda: {
            "name": "entropy",
            "entropy_type": "tsallis",
            "alpha": 0.5,
            "entropy_norm": "exp",
        })

    @dataclass
    class EndpointingConfig:
        stop_history_eou: int = 800
        residue_tokens_at_end: int = 2

    @dataclass
    class StreamingConfig:
        sample_rate: int = 16000
        batch_size: int = 512
        word_boundary_tolerance: int = 4
        att_context_size: List[int] = field(default_factory=lambda: [70, 6])
        use_cache: bool = True
        use_feat_cache: bool = True
        chunk_size_in_secs: Optional[float] = None
        request_type: str = "frame"
        num_slots: int = 1024
        exhaustive_batching: bool = False
        batching_delay_secs: float = 0.300

    @dataclass
    class MetricsASRConfig:
        gt_text_attr_name: str = "text"
        clean_groundtruth_text: bool = False
        langid: str = "hi"
        use_cer: bool = True
        ignore_capitalization: bool = True
        ignore_punctuation: bool = True


Output:
Chunk ID: e1cd79
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 3474
Output:
        ignore_punctuation: bool = True
        strip_punc_space: bool = False

    @dataclass
    class MetricsConfig:
        asr: MetricsASRConfig = field(default_factory=MetricsASRConfig)

    @dataclass
    class CacheAwarePipelineConfig:
        asr: ASRConfig = field(default_factory=ASRConfig)
        itn: ITNConfig = field(default_factory=ITNConfig)
        nmt: Optional[dict] = None
        confidence: ConfidenceConfig = field(default_factory=ConfidenceConfig)
        endpointing: EndpointingConfig = field(default_factory=EndpointingConfig)
        streaming: StreamingConfig = field(default_factory=StreamingConfig)
        matmul_precision: str = "high"
        log_level: int = 20
        pipeline_type: str = "cache_aware"
        asr_decoding_type: str = "rnnt"
        audio_file: Optional[str] = None
        output_filename: Optional[str] = None
        output_dir: Optional[str] = None
        enable_pnc: bool = False
        enable_itn: bool = False
        enable_nmt: bool = False
        asr_output_granularity: str = "segment"
        cache_dir: Optional[str] = None
        lang: Optional[str] = "hi"
        return_tail_result: bool = False
        calculate_wer: bool = True
        calculate_bleu: bool = True
        metrics: MetricsConfig = field(default_factory=MetricsConfig)

    config = CacheAwarePipelineConfig()
    config.asr.model_name = model_path
    return OmegaConf.structured(config)

async def load_model():
    """Load model and build pipeline. Called at startup."""
    from omegaconf import OmegaConf

    # Silence chatty NeMo logs
    logging.getLogger("nemo_logger").setLevel(logging.WARNING)

    logger.info("=== Starting model load ===")
    model_path = download_model_if_needed()
    state.cfg = build_pipeline_config(model_path)

    logger.info(f"Building pipeline: model={model_path}")
    logger.info(f"  att_context_size={state.cfg.streaming.att_context_size}")
    logger.info(f"  batch_size={state.cfg.streaming.batch_size}")

    # Build pipeline (reuses nemotron_asr pipeline code)
    from pipelines import StreamingPipelineBuilder
    state.pipeline = StreamingPipelineBuilder.build_pipeline(state.cfg)

    # Initialize streaming support
    state.pipeline.init_streaming_request_generator()
    logger.info("Streaming request generator initialized")

    # Warmup GPU (3 iterations)
    logger.info("Warming up GPU (3 iterations)...")
    from asr_utils import preprocess_audio
    from nemo.collections.asr.inference.streaming.framing.request_options import ASRRequestOptions

    WARMUP_URL = "https://github.com/voxserv/audio_quality_testing_samples/raw/refs/heads/master/mono_44100/156550__acclivity__a-dream-within-a-dream.wav"
    try:
        audio_bytes = preprocess_audio(WARMUP_URL, target_sample_rate=16000)
    except Exception as e:
        # If warmup audio fails to download, generate silence
        logger.warning(f"Warmup audio download failed: {e}, using silence")
        audio_bytes = np.zeros(16000 * 3, dtype=np.int16).tobytes()  # 3s silence

    chunk_size = int(0.080 * 16000 * 2)  # 80ms chunks, 16kHz, 16-bit

    for warmup_iter in range(1, 4):
        options = ASRRequestOptions()
        options.asr_output_granularity = "word"
        stream_id = state.pipeline.open_streaming_session(options=options)

        for i in range(0, len(audio_bytes), chunk_size):
            chunk = audio_bytes[i:i + chunk_size]
            state.pipeline.append_streaming_audio(stream_id, chunk)
            await state.pipeline.process_streaming_batch()

        state.pipeline.close_streaming_session(stream_id)
        try:
            await state.pipeline.process_streaming_batch()
        except Exception:
            pass

        # Cleanup warmup stream
        if stream_id in state.pipeline._state_pool:
            del state.pipeline._state_pool[stream_id]
        if stream_id in state.pipeline._streaming_request_generator.streams:
            state.pipeline._streaming_request_generator.streams.pop(stream_id, None)

        logger.info(f"  Warmup {warmup_iter}/3 complete")

    # Initialize VAD
    if ENABLE_VAD:
        from vad import SileroVAD
        state.vad = SileroVAD(threshold=VAD_THRESHOLD, sample_rate=16000)
        logger.info(f"VAD enabled (threshold={VAD_THRESHOLD})")
    else:
        logger.info("VAD disabled")

    state.start_time = time.perf_counter()
    state.model_loaded = True

    # CRITICAL: This log line triggers Vast.ai PyWorker readiness detection
    logger.info("Application startup complete.")
    print("Application startup complete.", flush=True)

async def transcribe_pcm_bytes_streaming(pcm_bytes: bytes) -> str:
    """
    Transcribe PCM bytes using the streaming pipeline (no .run() required).
    WHY: CacheAwareRNNTStreamingPipeline doesn't implement .run().
    """
    from nemo.collections.asr.inference.streaming.framing.request_options import ASRRequestOptions

    options = ASRRequestOptions()
    options.asr_output_granularity = "word"
    stream_id = state.pipeline.open_streaming_session(options=options)

    # Use pipeline's native chunk size if available; fallback to 80ms
    frame_secs = getattr(state.pipeline, "chunk_size_in_secs", 0.08) or 0.08
    frame_bytes = max(2560, int(frame_secs * 16000 * 2))  # 16kHz, 16-bit

    full_segments: list[str] = []
    last_partial = ""

    try:
        # Feed audio in frame-sized chunks
        for i in range(0, len(pcm_bytes), frame_bytes):
            chunk = pcm_bytes[i:i + frame_bytes]
            state.pipeline.append_streaming_audio(stream_id, chunk)
            try:
                requests, outputs = await state.pipeline.process_streaming_batch()
            except Exception as e:
                if type(e).__name__ == "NotEnoughDataException":
                    outputs = []
                else:
                    raise
            for output in outputs:
                if output.stream_id != stream_id:
                    continue
                if output.final_transcript:
                    full_segments.append(output.final_transcript.strip())
                    # Final transcript commits buffered tokens; clear stale partial.
                    last_partial = ""
                if output.partial_transcript:
                    last_partial = output.partial_transcript.strip()

        # Close and drain remaining outputs
        state.pipeline.close_streaming_session(stream_id)
        for _ in range(10):
            try:
                requests, outputs = await state.pipeline.process_streaming_batch()
            except Exception as e:
                if type(e).__name__ == "NotEnoughDataException":
                    outputs = []
                else:
                    break
            if not outputs:
                break
            for output in outputs:
                if output.stream_id != stream_id:
                    continue
                if output.final_transcript:
                    full_segments.append(output.final_transcript.strip())
                    last_partial = ""
                if output.partial_transcript:
                    last_partial = output.partial_transcript.strip()

    finally:
        # Cleanup stream state
        if stream_id in state.pipeline._state_pool:
            del state.pipeline._state_pool[stream_id]
        if stream_id in state.pipeline._streaming_request_generator.streams:
            state.pipeline._streaming_request_generator.streams.pop(stream_id, None)

    # Keep tail partial to avoid truncating long utterances with no final EOU flush.
    combined = " ".join([x for x in [*full_segments, last_partial] if x]).strip()
    final_only = " ".join([x for x in full_segments if x]).strip()
    if len(combined) >= len(final_only):
        return combined
    return final_only or last_partial

# ============================================================
# HTTP Endpoints
# ============================================================

async def _parse_audio_request(request: Request):
    """
    Parse audio payload from either JSON (audio_base64) or multipart form-data (file).
    Returns (audio_bytes, language, response_format, chunk_duration_ms).
    """
    content_type = request.headers.get("content-type", "")

    if content_type.startswith("application/json"):
        body = await request.json()
        audio_b64 = (
            body.get("audio_base64")
            or body.get("audio_b64")
            or body.get("audio")
        )
        if not audio_b64:
            raise HTTPException(status_code=400, detail="audio_base64 required")
        try:
            audio_data = base64.b64decode(audio_b64)
        except Exception as e:
            raise HTTPException(status_code=400, detail=f"invalid audio_base64: {e}")
        language = body.get("language", "hi")
        response_format = body.get("response_format", "json")
        chunk_duration_ms = int(body.get("chunk_duration_ms", 160) or 160)
        return audio_data, language, response_format, chunk_duration_ms

    # Fallback: multipart form-data
    form = await request.form()
    file_obj = form.get("file")
    if file_obj is None:
        raise HTTPException(status_code=400, detail="file required")

    if hasattr(file_obj, "read"):
        audio_data = await file_obj.read()
    else:
        audio_data = file_obj

    language = form.get("language", "hi")
    response_format = form.get("response_format", "json")
    chunk_duration_ms = int(form.get("chunk_duration_ms", 160) or 160)
    return audio_data, language, response_format, chunk_duration_ms

@app.on_event("startup")
async def startup():
    """Load model on server startup"""
    await load_model()

@app.get("/health")
async def health_check():
    """
    Health check for load balancers and PyWorker.
    Returns 503 when overloaded so /route/ skips this worker.
    """
    try:
        vram_used = torch.cuda.memory_allocated() / 1e9
        vram_total = torch.cuda.get_device_properties(0).total_memory / 1e9
        gpu_name = torch.cuda.get_device_name(0)
    except Exception:
        vram_used = vram_total = 0
        gpu_name = "unknown"

    active = len(state.client_queues)
    status = "healthy" if active < MAX_ACTIVE_STREAMS else "overloaded"
    uptime = round(time.perf_counter() - state.start_time, 0) if state.start_time else 0

    return JSONResponse(
        status_code=200 if status == "healthy" else 503,
        content={
            "status": status,
            "model_loaded": state.model_loaded,
            "active_streams": active,
            "max_streams": MAX_ACTIVE_STREAMS,
            "gpu": gpu_name,
            "vram_used_gb": round(vram_used, 2),
            "vram_total_gb": round(vram_total, 0),
            "inference_running": state.inference_running,
            "uptime_s": uptime,
        },
    )

def _build_metrics_payload():
    """Shared metrics payload for GET/POST /metrics."""
    m = state.metrics

    def pct(arr, p):
        if not arr:
            return 0
        s = sorted(arr)
        idx = min(int(len(s) * p / 100), len(s) - 1)
        return round(s[idx], 2)

    cuda_available = False
    device_count = 0
    gpu_name = "unknown"
    vram_used = vram_total = vram_reserved = 0
    try:
        cuda_available = torch.cuda.is_available()
        device_count = torch.cuda.device_count() if cuda_available else 0
        if cuda_available and device_count > 0:
            gpu_name = torch.cuda.get_device_name(0)
            vram_total = torch.cuda.get_device_properties(0).total_memory / 1e9
            vram_used = torch.cuda.memory_allocated() / 1e9
            vram_reserved = torch.cuda.memory_reserved() / 1e9
    except Exception:
        pass

    return {
        "active_streams": len(state.client_queues),
        "max_streams": MAX_ACTIVE_STREAMS,
        "inference": {
            "total_batches": m["total_batches"],
            "total_outputs": m["total_outputs"],
            "empty_cycles": m["empty_cycles"],
            "vad_filtered_chunks": m["vad_filtered_chunks"],
            "gpu_ms_p50": pct(m["gpu_times_ms"], 50),
            "gpu_ms_p99": pct(m["gpu_times_ms"], 99),
            "cycle_ms_p50": pct(m["cycle_times_ms"], 50),
            "cycle_ms_p99": pct(m["cycle_times_ms"], 99),
            "batch_size_avg": round(sum(m["batch_sizes"]) / max(len(m["batch_sizes"]), 1), 1),
        },
        "http": {
            "requests_total": m["http_requests_total"],
            "requests_ok": m["http_requests_ok"],
            "requests_err": m["http_requests_err"],
        },
        "ws": {
            "connections_total": m["ws_connections_total"],
            "connections_active": m["ws_connections_active"],
            "connections_rejected": m["ws_connections_rejected"],
        },
        "cuda_available": cuda_available,
        "cuda_device_count": device_count,
        "gpu_name": gpu_name,
        "vram_used_gb": round(vram_used, 2),
        "vram_reserved_gb": round(vram_reserved, 2),
        "vram_total_gb": round(vram_total, 0),
        "uptime_s": round(time.perf_counter() - state.start_time, 0) if state.start_time else 0,
    }

@app.get("/metrics")
async def metrics_get():
    """
    Detailed metrics endpoint for monitoring/alerting.
    Returns structured JSON with rolling-window percentiles.
    """
    return _build_metrics_payload()

@app.post("/metrics")
async def metrics_post(_request: Request):
    """POST variant for serverless PyWorker proxy."""
    return _build_metrics_payload()

@app.post("/v1/audio/transcriptions/json")
async def transcribe_json(request: Request):
    """
    JSON-based transcription endpoint for PyWorker benchmarks.
    Accepts: {"audio_base64": "...", "language": "hi"}
    WHY: PyWorker sends benchmark payloads as JSON, not multipart.
    """
    state.metrics["http_requests_total"] += 1
    if not state.model_loaded:


**Inspecting WebSocket endpoint details**


Output:
Chunk ID: 0b1bc3
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2769
Output:
                            break
                        for output in outputs:
                            if output.stream_id == stream_id:
                                text = output.final_transcript or output.partial_transcript or ""
                                is_final = bool(output.final_transcript)
                                if text.strip():
                                    if is_final:
                                        full_segments.append(text.strip())
                                        last_partial = ""
                                    else:
                                        last_partial = text.strip()
                                    event_data = json.dumps({
                                        "text": text,
                                        "is_final": is_final,
                                        "timestamp": time.time(),
                                    })
                                    yield f"data: {event_data}\n\n"
                    except Exception:
                        break

                # Final event with complete transcription
                combined = " ".join([x for x in [*full_segments, last_partial] if x]).strip()
                yield f"data: {json.dumps({'text': combined, 'is_final': True, 'done': True})}\n\n"

            finally:
                # Cleanup
                if stream_id in state.pipeline._state_pool:
                    del state.pipeline._state_pool[stream_id]
                if stream_id in state.pipeline._streaming_request_generator.streams:
                    state.pipeline._streaming_request_generator.streams.pop(stream_id, None)

        state.metrics["http_requests_ok"] += 1
        return StreamingResponse(
            sse_generator(),
            media_type="text/event-stream",
            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
        )

    except HTTPException:
        raise
    except Exception as e:
        state.metrics["http_requests_err"] += 1
        logger.error(f"HTTP stream error: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail=str(e))

# ============================================================
# WebSocket Streaming
# ============================================================

@app.websocket("/ws")
async def websocket_streaming(ws: WebSocket):
    """
    Production WebSocket endpoint for real-time streaming ASR.

    Protocol:
      Client -> Server: binary audio chunks (raw PCM 16-bit 16kHz mono)
      Client -> Server: text "END" to signal end of audio
      Server -> Client: JSON {"text": "...", "is_final": bool, "timestamp": float}
      Server -> Client: text "END" to confirm stream complete

    Backpressure: Rejects with 1013 when at capacity.
    """
    state.metrics["ws_connections_total"] += 1

    # Backpressure: fast-fail when overloaded
    active = len(state.client_queues)
    if active >= MAX_ACTIVE_STREAMS:
        state.metrics["ws_connections_rejected"] += 1
        await ws.accept()
        await ws.close(code=1013, reason=f"Overloaded: {active}/{MAX_ACTIVE_STREAMS}")
        return

    await ws.accept()
    state.metrics["ws_connections_active"] += 1

    stream_id = None
    try:
        # Start inference loop if needed
        start_inference_loop_if_needed()

        # Open streaming session
        from nemo.collections.asr.inference.streaming.framing.request_options import ASRRequestOptions
        options = ASRRequestOptions()
        options.asr_output_granularity = "word"
        stream_id = state.pipeline.open_streaming_session(options=options)
        transcription_queue = await register_client(stream_id)
        elapsed = time.perf_counter() - state.start_time
        logger.info(f"[+{elapsed:.1f}s] WS client {stream_id} connected ({len(state.client_queues)} total)")

    except Exception as e:
        logger.error(f"WS connection setup error: {e}", exc_info=True)
        state.metrics["ws_connections_active"] -= 1
        if stream_id is not None:
            await unregister_client(stream_id)
        if ws.application_state is WebSocketState.CONNECTED:
            await ws.close()
        return

    # Send ready message
    await ws.send_text(json.dumps({"type": "Ready", "id": stream_id}))
    recv_done = asyncio.Event()

    async def recv_loop():
        """Receive audio chunks from client, feed to pipeline"""
        att_ctx = state.cfg.streaming.att_context_size[1] + 1
        frame_threshold = att_ctx * 1280 * 2  # bytes
        audio_buffer = b""

        try:
            while True:
                message = await ws.receive()

                # Text message = control signal
                if "text" in message:
                    text_msg = message["text"]
                    if text_msg == "END":
                        break

                # Binary message = audio data
                if "bytes" in message:
                    raw_bytes = message["bytes"]
                    if isinstance(raw_bytes, str):
                        raw_bytes = raw_bytes.encode("latin-1")
                    audio_buffer += raw_bytes

                    # Feed to pipeline when buffer exceeds frame threshold
                    if len(audio_buffer) >= frame_threshold:
                        # VAD gate: skip silence before GPU
                        if state.vad is not None and not state.vad.is_speech(audio_buffer, stream_id):
                            state.metrics["vad_filtered_chunks"] += 1
                            audio_buffer = b""
                            continue

                        state.pipeline.append_streaming_audio(stream_id, audio_buffer)
                        audio_buffer = b""

                        # Signal inference loop
                        state._batch_ready_event.set()
                        await asyncio.sleep(0)

        except WebSocketDisconnect:
            pass
        except Exception as e:
            logger.error(f"WS recv error stream {stream_id}: {e}")
        finally:
            # Flush remaining buffer
            if audio_buffer:
                try:
                    state.pipeline.append_streaming_audio(stream_id, audio_buffer)
                except Exception:
                    pass

            # Mark stream as ended
            try:
                state.pipeline._streaming_request_generator.streams[stream_id].mark_end()
            except Exception:
                pass

            # Drain pending outputs before terminating sender.
            drain_deadline = time.perf_counter() + 8.0
            while time.perf_counter() < drain_deadline:
                try:
                    stream_alive = stream_id in state.pipeline._streaming_request_generator.streams
                except Exception:
                    stream_alive = False
                if not stream_alive:
                    break
                state._batch_ready_event.set()
                await asyncio.sleep(0.02)

            state._batch_ready_event.set()
            recv_done.set()

    async def send_loop():
        """Send transcription results to client"""
        step_num = 0
        idle_ticks_after_recv = 0
        try:
            while True:
                try:
                    output_dict = await asyncio.wait_for(transcription_queue.get(), timeout=0.2)
                    idle_ticks_after_recv = 0
                except asyncio.TimeoutError:
                    if recv_done.is_set():
                        idle_ticks_after_recv += 1
                        if idle_ticks_after_recv >= 5:
                            await ws.send_text("END")
                            break
                    continue

                json_str = output_dict.get("json_str")
                if json_str:
                    if step_num == 0:
                        elapsed = time.perf_counter() - state.start_time
                        logger.info(f"[+{elapsed:.1f}s] Stream {stream_id}: first token sent")
                    await ws.send_text(json_str)
                    step_num += 1

        except Exception as e:
            logger.error(f"WS send error stream {stream_id}: {e}")

    tasks = []
    try:
        tasks = [
            asyncio.create_task(recv_loop()),
            asyncio.create_task(send_loop()),
        ]
        await asyncio.gather(*tasks, return_exceptions=True)
    except Exception as e:
        logger.error(f"WS error stream {stream_id}: {e}")
    finally:
        state.metrics["ws_connections_active"] -= 1
        await unregister_client(stream_id)
        if ws.application_state is WebSocketState.CONNECTED:
            await ws.close()
        for task in tasks:
            if not task.done():
                task.cancel()
                try:
                    await task
                except asyncio.CancelledError:
                    pass

# ============================================================
# Stream Management + Inference Loop
# ============================================================

async def register_client(stream_id: int) -> asyncio.Queue:
    """Register a streaming client, return its output queue"""
    queue = asyncio.Queue()
    async with state.client_queues_lock:
        state.client_queues[stream_id] = queue
    async with state.stream_timestamps_lock:
        state.stream_audio_timestamps[stream_id] = 0.0
    return queue

async def unregister_client(stream_id: int):
    """Remove client from tracking, mark for cleanup"""
    try:
        async with state.client_queues_lock:
            state.client_queues.pop(stream_id, None)
        async with state.stream_timestamps_lock:
            state.stream_audio_timestamps.pop(stream_id, None)
        if state.vad is not None:
            state.vad.reset_stream(stream_id)
        if hasattr(state.pipeline, "_streaming_request_generator"):
            state.pipeline._streaming_request_generator.close_stream(stream_id)
        async with state.streams_cleanup_lock:
            state.streams_pending_cleanup.add(stream_id)
    except Exception as e:
        logger.error(f"Unregister error stream {stream_id}: {e}")

async def route_outputs(outputs_with_timestamps):
    """Route inference outputs to per-client queues"""
    current_time = time.time()

    for output_dict in outputs_with_timestamps:
        output = output_dict["output"]
        audio_ts = output_dict["audio_timestamp"]

        text = output.current_step_transcript if output.partial_transcript else output.final_transcript
        is_final = not bool(output.partial_transcript)

        result = {
            "text": text or "",
            "is_final": is_final,
            "timestamp": current_time,
            "audio_timestamp": audio_ts,
        }

        if len(output.final_segments) > 0:
            result["segment_text"] = output.final_segments[0].text
            result["segment_start"] = output.final_segments[0].start
            result["segment_end"] = output.final_segments[0].end

        output_dict["json_str"] = json.dumps(result)


**Inspecting inference loop and batching**


sed -n '1060,1320p' production/server.py