Output:
Chunk ID: fb8c5b
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 5170
Output:
        ignore_punctuation: bool = True
        strip_punc_space: bool = False

    @dataclass
    class MetricsConfig:
        asr: MetricsASRConfig = field(default_factory=MetricsASRConfig)

    @dataclass
    class CacheAwarePipelineConfig:
        asr: ASRConfig = field(default_factory=ASRConfig)
        itn: ITNConfig = field(default_factory=ITNConfig)
        nmt: Optional[dict] = None
        confidence: ConfidenceConfig = field(default_factory=ConfidenceConfig)
        endpointing: EndpointingConfig = field(default_factory=EndpointingConfig)
        streaming: StreamingConfig = field(default_factory=StreamingConfig)
        matmul_precision: str = "high"
        log_level: int = 20
        pipeline_type: str = "cache_aware"
        asr_decoding_type: str = "rnnt"
        audio_file: Optional[str] = None
        output_filename: Optional[str] = None
        output_dir: Optional[str] = None
        enable_pnc: bool = False
        enable_itn: bool = False
        enable_nmt: bool = False
        asr_output_granularity: str = "segment"
        cache_dir: Optional[str] = None
        lang: Optional[str] = "hi"
        return_tail_result: bool = False
        calculate_wer: bool = True
        calculate_bleu: bool = True
        metrics: MetricsConfig = field(default_factory=MetricsConfig)

    config = CacheAwarePipelineConfig()
    config.asr.model_name = model_path
    return OmegaConf.structured(config)

async def load_model():
    """Load model and build pipeline. Called at startup."""
    from omegaconf import OmegaConf

    # Silence chatty NeMo logs
    logging.getLogger("nemo_logger").setLevel(logging.WARNING)

    logger.info("=== Starting model load ===")
    model_path = download_model_if_needed()
    state.cfg = build_pipeline_config(model_path)

    logger.info(f"Building pipeline: model={model_path}")
    logger.info(f"  att_context_size={state.cfg.streaming.att_context_size}")
    logger.info(f"  batch_size={state.cfg.streaming.batch_size}")

    # Build pipeline (reuses nemotron_asr pipeline code)
    from pipelines import StreamingPipelineBuilder
    state.pipeline = StreamingPipelineBuilder.build_pipeline(state.cfg)

    # Initialize streaming support
    state.pipeline.init_streaming_request_generator()
    logger.info("Streaming request generator initialized")

    # Warmup GPU (3 iterations)
    logger.info("Warming up GPU (3 iterations)...")
    from asr_utils import preprocess_audio
    from nemo.collections.asr.inference.streaming.framing.request_options import ASRRequestOptions

    WARMUP_URL = "https://github.com/voxserv/audio_quality_testing_samples/raw/refs/heads/master/mono_44100/156550__acclivity__a-dream-within-a-dream.wav"
    try:
        audio_bytes = preprocess_audio(WARMUP_URL, target_sample_rate=16000)
    except Exception as e:
        # If warmup audio fails to download, generate silence
        logger.warning(f"Warmup audio download failed: {e}, using silence")
        audio_bytes = np.zeros(16000 * 3, dtype=np.int16).tobytes()  # 3s silence

    chunk_size = int(0.080 * 16000 * 2)  # 80ms chunks, 16kHz, 16-bit

    for warmup_iter in range(1, 4):
        options = ASRRequestOptions()
        options.asr_output_granularity = "word"
        stream_id = state.pipeline.open_streaming_session(options=options)

        for i in range(0, len(audio_bytes), chunk_size):
            chunk = audio_bytes[i:i + chunk_size]
            state.pipeline.append_streaming_audio(stream_id, chunk)
            await state.pipeline.process_streaming_batch()

        state.pipeline.close_streaming_session(stream_id)
        try:
            await state.pipeline.process_streaming_batch()
        except Exception:
            pass

        # Cleanup warmup stream
        if stream_id in state.pipeline._state_pool:
            del state.pipeline._state_pool[stream_id]
        if stream_id in state.pipeline._streaming_request_generator.streams:
            state.pipeline._streaming_request_generator.streams.pop(stream_id, None)

        logger.info(f"  Warmup {warmup_iter}/3 complete")

    # Initialize VAD
    if ENABLE_VAD:
        from vad import SileroVAD
        state.vad = SileroVAD(threshold=VAD_THRESHOLD, sample_rate=16000)
        logger.info(f"VAD enabled (threshold={VAD_THRESHOLD})")
    else:
        logger.info("VAD disabled")

    state.start_time = time.perf_counter()
    state.model_loaded = True

    # CRITICAL: This log line triggers Vast.ai PyWorker readiness detection
    logger.info("Application startup complete.")
    print("Application startup complete.", flush=True)

async def transcribe_pcm_bytes_streaming(pcm_bytes: bytes) -> str:
    """
    Transcribe PCM bytes using the streaming pipeline (no .run() required).
    WHY: CacheAwareRNNTStreamingPipeline doesn't implement .run().
    """
    from nemo.collections.asr.inference.streaming.framing.request_options import ASRRequestOptions

    options = ASRRequestOptions()
    options.asr_output_granularity = "word"
    stream_id = state.pipeline.open_streaming_session(options=options)

    # Use pipeline's native chunk size if available; fallback to 80ms
    frame_secs = getattr(state.pipeline, "chunk_size_in_secs", 0.08) or 0.08
    frame_bytes = max(2560, int(frame_secs * 16000 * 2))  # 16kHz, 16-bit

    full_text = ""
    last_partial = ""

    try:
        # Feed audio in frame-sized chunks
        for i in range(0, len(pcm_bytes), frame_bytes):
            chunk = pcm_bytes[i:i + frame_bytes]
            state.pipeline.append_streaming_audio(stream_id, chunk)
            try:
                requests, outputs = await state.pipeline.process_streaming_batch()
            except Exception as e:
                if type(e).__name__ == "NotEnoughDataException":
                    outputs = []
                else:
                    raise
            for output in outputs:
                if output.stream_id != stream_id:
                    continue
                if output.final_transcript:
                    full_text += output.final_transcript
                if output.partial_transcript:
                    last_partial = output.partial_transcript

        # Close and drain remaining outputs
        state.pipeline.close_streaming_session(stream_id)
        for _ in range(10):
            try:
                requests, outputs = await state.pipeline.process_streaming_batch()
            except Exception as e:
                if type(e).__name__ == "NotEnoughDataException":
                    outputs = []
                else:
                    break
            if not outputs:
                break
            for output in outputs:
                if output.stream_id != stream_id:
                    continue
                if output.final_transcript:
                    full_text += output.final_transcript
                if output.partial_transcript:
                    last_partial = output.partial_transcript

    finally:
        # Cleanup stream state
        if stream_id in state.pipeline._state_pool:
            del state.pipeline._state_pool[stream_id]
        if stream_id in state.pipeline._streaming_request_generator.streams:
            state.pipeline._streaming_request_generator.streams.pop(stream_id, None)

    return full_text.strip() or last_partial.strip()

# ============================================================
# HTTP Endpoints
# ============================================================

async def _parse_audio_request(request: Request):
    """
    Parse audio payload from either JSON (audio_base64) or multipart form-data (file).
    Returns (audio_bytes, language, response_format, chunk_duration_ms).
    """
    content_type = request.headers.get("content-type", "")

    if content_type.startswith("application/json"):
        body = await request.json()
        audio_b64 = (
            body.get("audio_base64")
            or body.get("audio_b64")
            or body.get("audio")
        )
        if not audio_b64:
            raise HTTPException(status_code=400, detail="audio_base64 required")
        try:
            audio_data = base64.b64decode(audio_b64)
        except Exception as e:
            raise HTTPException(status_code=400, detail=f"invalid audio_base64: {e}")
        language = body.get("language", "hi")
        response_format = body.get("response_format", "json")
        chunk_duration_ms = int(body.get("chunk_duration_ms", 160) or 160)
        return audio_data, language, response_format, chunk_duration_ms

    # Fallback: multipart form-data
    form = await request.form()
    file_obj = form.get("file")
    if file_obj is None:
        raise HTTPException(status_code=400, detail="file required")

    if hasattr(file_obj, "read"):
        audio_data = await file_obj.read()
    else:
        audio_data = file_obj

    language = form.get("language", "hi")
    response_format = form.get("response_format", "json")
    chunk_duration_ms = int(form.get("chunk_duration_ms", 160) or 160)
    return audio_data, language, response_format, chunk_duration_ms

@app.on_event("startup")
async def startup():
    """Load model on server startup"""
    await load_model()

@app.get("/health")
async def health_check():
    """
    Health check for load balancers and PyWorker.
    Returns 503 when overloaded so /route/ skips this worker.
    """
    try:
        vram_used = torch.cuda.memory_allocated() / 1e9
        vram_total = torch.cuda.get_device_properties(0).total_memory / 1e9
        gpu_name = torch.cuda.get_device_name(0)
    except Exception:
        vram_used = vram_total = 0
        gpu_name = "unknown"

    active = len(state.client_queues)
    status = "healthy" if active < MAX_ACTIVE_STREAMS else "overloaded"
    uptime = round(time.perf_counter() - state.start_time, 0) if state.start_time else 0

    return JSONResponse(
        status_code=200 if status == "healthy" else 503,
        content={
            "status": status,
            "model_loaded": state.model_loaded,
            "active_streams": active,
            "max_streams": MAX_ACTIVE_STREAMS,
            "gpu": gpu_name,
            "vram_used_gb": round(vram_used, 2),
            "vram_total_gb": round(vram_total, 0),
            "inference_running": state.inference_running,
            "uptime_s": uptime,
        },
    )

def _build_metrics_payload():
    """Shared metrics payload for GET/POST /metrics."""
    m = state.metrics

    def pct(arr, p):
        if not arr:
            return 0
        s = sorted(arr)
        idx = min(int(len(s) * p / 100), len(s) - 1)
        return round(s[idx], 2)

    cuda_available = False
    device_count = 0
    gpu_name = "unknown"
    vram_used = vram_total = vram_reserved = 0
    try:
        cuda_available = torch.cuda.is_available()
        device_count = torch.cuda.device_count() if cuda_available else 0
        if cuda_available and device_count > 0:
            gpu_name = torch.cuda.get_device_name(0)
            vram_total = torch.cuda.get_device_properties(0).total_memory / 1e9
            vram_used = torch.cuda.memory_allocated() / 1e9
            vram_reserved = torch.cuda.memory_reserved() / 1e9
    except Exception:
        pass

    return {
        "active_streams": len(state.client_queues),
        "max_streams": MAX_ACTIVE_STREAMS,
        "inference": {
            "total_batches": m["total_batches"],
            "total_outputs": m["total_outputs"],
            "empty_cycles": m["empty_cycles"],
            "vad_filtered_chunks": m["vad_filtered_chunks"],
            "gpu_ms_p50": pct(m["gpu_times_ms"], 50),
            "gpu_ms_p99": pct(m["gpu_times_ms"], 99),
            "cycle_ms_p50": pct(m["cycle_times_ms"], 50),
            "cycle_ms_p99": pct(m["cycle_times_ms"], 99),
            "batch_size_avg": round(sum(m["batch_sizes"]) / max(len(m["batch_sizes"]), 1), 1),
        },
        "http": {
            "requests_total": m["http_requests_total"],
            "requests_ok": m["http_requests_ok"],
            "requests_err": m["http_requests_err"],
        },
        "ws": {
            "connections_total": m["ws_connections_total"],
            "connections_active": m["ws_connections_active"],
            "connections_rejected": m["ws_connections_rejected"],
        },
        "cuda_available": cuda_available,
        "cuda_device_count": device_count,
        "gpu_name": gpu_name,
        "vram_used_gb": round(vram_used, 2),
        "vram_reserved_gb": round(vram_reserved, 2),
        "vram_total_gb": round(vram_total, 0),
        "uptime_s": round(time.perf_counter() - state.start_time, 0) if state.start_time else 0,
    }

@app.get("/metrics")
async def metrics_get():
    """
    Detailed metrics endpoint for monitoring/alerting.
    Returns structured JSON with rolling-window percentiles.
    """
    return _build_metrics_payload()

@app.post("/metrics")
async def metrics_post(_request: Request):
    """POST variant for serverless PyWorker proxy."""
    return _build_metrics_payload()

@app.post("/v1/audio/transcriptions/json")
async def transcribe_json(request: Request):
    """
    JSON-based transcription endpoint for PyWorker benchmarks.
    Accepts: {"audio_base64": "...", "language": "hi"}
    WHY: PyWorker sends benchmark payloads as JSON, not multipart.
    """
    state.metrics["http_requests_total"] += 1
    if not state.model_loaded:
        state.metrics["http_requests_err"] += 1
        raise HTTPException(status_code=503, detail="Model not loaded yet")
    t_start = time.perf_counter()
    try:
        audio_data, language, _response_format, _chunk_duration_ms = await _parse_audio_request(request)
        from asr_utils import preprocess_audio
        pcm_bytes = preprocess_audio(audio_data, target_sample_rate=16000)
        audio_duration = len(pcm_bytes) / (16000 * 2)
        t_infer = time.perf_counter()
        text = await transcribe_pcm_bytes_streaming(pcm_bytes)
        t_done = time.perf_counter()
        state.metrics["http_requests_ok"] += 1
        return {
            "text": text,
            "duration": round(audio_duration, 2),
            "processing_time": round(t_done - t_infer, 3),
        }
    except HTTPException:
        raise
    except Exception as e:
        state.metrics["http_requests_err"] += 1
        logger.error(f"JSON transcribe error: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/v1/audio/transcriptions")
async def transcribe_batch(request: Request):
    """
    OpenAI-compatible batch transcription endpoint.
    Accepts audio file, returns full transcription.

    WHY HTTP batch: For non-real-time use cases (file transcription, post-processing).
    Uses pipeline.run() directly — simpler, no streaming overhead.
    """
    state.metrics["http_requests_total"] += 1

    if not state.model_loaded:
        state.metrics["http_requests_err"] += 1
        raise HTTPException(status_code=503, detail="Model not loaded yet")

    t_start = time.perf_counter()

    try:
        # Read uploaded audio (multipart or JSON base64)
        audio_data, language, response_format, _chunk_duration_ms = await _parse_audio_request(request)
        if not audio_data:
            raise HTTPException(status_code=400, detail="Empty audio file")

        # Preprocess audio to 16kHz mono PCM
        from asr_utils import preprocess_audio
        pcm_bytes = preprocess_audio(audio_data, target_sample_rate=16000)
        audio_duration = len(pcm_bytes) / (16000 * 2)  # seconds

        # Run streaming inference (batch)
        t_infer = time.perf_counter()
        text = await transcribe_pcm_bytes_streaming(pcm_bytes)
        t_done = time.perf_counter()

        result = {
            "text": text,
            "language": language,
            "duration": round(audio_duration, 2),
            "processing_time": round(t_done - t_infer, 3),
            "total_time": round(t_done - t_start, 3),
        }

        state.metrics["http_requests_ok"] += 1
        logger.info(f"HTTP batch: {audio_duration:.1f}s audio -> {t_done - t_infer:.2f}s inference")
        return result

    except HTTPException:
        raise
    except Exception as e:
        state.metrics["http_requests_err"] += 1
        logger.error(f"HTTP batch error: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/v1/audio/transcriptions/stream")
async def transcribe_stream(request: Request):
    """
    HTTP streaming transcription via Server-Sent Events (SSE).
    Client uploads audio file, gets progressive transcription events.

    WHY SSE: Works through HTTP proxies (including Vast.ai PyWorker).
    Compatible with standard HTTP clients — no WebSocket library needed.
    """
    state.metrics["http_requests_total"] += 1

    if not state.model_loaded:
        state.metrics["http_requests_err"] += 1
        raise HTTPException(status_code=503, detail="Model not loaded yet")

    try:
        audio_data, language, response_format, chunk_duration_ms = await _parse_audio_request(request)
        if not audio_data:
            raise HTTPException(status_code=400, detail="Empty audio file")

        from asr_utils import preprocess_audio
        pcm_bytes = preprocess_audio(audio_data, target_sample_rate=16000)

        async def sse_generator():
            """Stream transcription results as SSE events"""
            from nemo.collections.asr.inference.streaming.framing.request_options import ASRRequestOptions

            options = ASRRequestOptions()
            options.asr_output_granularity = "word"
            stream_id = state.pipeline.open_streaming_session(options=options)

            chunk_size = int(chunk_duration_ms / 1000.0 * 16000 * 2)  # bytes per chunk
            att_ctx = state.cfg.streaming.att_context_size[1] + 1
            frame_bytes = att_ctx * 1280 * 2  # samples per frame * 2 bytes

            audio_buffer = b""
            full_text = ""

            try:
                for i in range(0, len(pcm_bytes), chunk_size):
                    chunk = pcm_bytes[i:i + chunk_size]
                    audio_buffer += chunk

                    # Feed to pipeline when we have enough for a frame
                    if len(audio_buffer) >= frame_bytes:
                        state.pipeline.append_streaming_audio(stream_id, audio_buffer)
                        audio_buffer = b""

                        # Process batch
                        requests, outputs = await state.pipeline.process_streaming_batch()
                        for output in outputs:
                            if output.stream_id == stream_id:
                                text = output.final_transcript or output.partial_transcript or ""
                                is_final = bool(output.final_transcript)
                                if text.strip():
                                    if is_final:
                                        full_text += text
                                    event_data = json.dumps({
                                        "text": text,
                                        "is_final": is_final,
                                        "timestamp": time.time(),
                                    })
                                    yield f"data: {event_data}\n\n"

                    # Simulate real-time pacing (optional, for more realistic SSE)
                    await asyncio.sleep(0)

                # Flush remaining audio
                if audio_buffer:
                    state.pipeline.append_streaming_audio(stream_id, audio_buffer)

                # Close stream and drain remaining outputs
                state.pipeline.close_streaming_session(stream_id)
                for _ in range(10):
                    try:
                        requests, outputs = await state.pipeline.process_streaming_batch()
                        if not outputs:
                            break
                        for output in outputs:
                            if output.stream_id == stream_id:
                                text = output.final_transcript or output.partial_transcript or ""
                                is_final = bool(output.final_transcript)
                                if text.strip():
                                    if is_final:
                                        full_text += text
                                    event_data = json.dumps({
                                        "text": text,
                                        "is_final": is_final,
                                        "timestamp": time.time(),
                                    })


Output:
Chunk ID: 210ab4
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 3285
Output:
#!/usr/bin/env python3
"""
Smoke test for Hindi ASR production server.
Tests: health, HTTP batch, HTTP SSE stream, WebSocket streaming.
Usage:
  python test_smoke.py --url http://worker-ip:8000
  python test_smoke.py --endpoint hindi-asr-prod --route-cost 10
"""
import os, sys, json, time, struct, asyncio, argparse, math, base64
from pathlib import Path
import requests
from urllib.parse import urlparse
import aiohttp

def generate_test_wav(duration_s=3.0, freq_hz=440, sample_rate=16000):
    """Generate sine wave as WAV bytes + raw PCM"""
    n = int(sample_rate * duration_s)
    pcm = struct.pack(f'<{n}h', *[min(32767, max(-32768, int(16000*math.sin(2*math.pi*freq_hz*i/sample_rate)))) for i in range(n)])
    hdr = struct.pack('<4sI4s4sIHHIIHH4sI', b'RIFF', 36+len(pcm), b'WAVE', b'fmt ', 16, 1, 1, sample_rate, sample_rate*2, 2, 16, b'data', len(pcm))
    return pcm, hdr + pcm

VAST_API = "https://console.vast.ai/api/v0"
VAST_RUN = "https://run.vast.ai"

def _env_or_dotenv(name: str):
    if os.environ.get(name):
        return os.environ.get(name)
    env_path = Path(__file__).resolve().parent.parent / ".env"
    if env_path.exists():
        for line in env_path.read_text().splitlines():
            if line.startswith(name + "="):
                return line.split("=", 1)[1].strip()
    return None

def get_endpoint_info(endpoint_name: str):
    account_key = _env_or_dotenv("VAST_KEY")
    if not account_key:
        raise RuntimeError("VAST_KEY not set; pass --endpoint-key")
    r = requests.get(
        f"{VAST_API}/endptjobs/",
        headers={"Accept": "application/json", "Authorization": f"Bearer {account_key}"},
        timeout=30,
    )
    data = r.json()
    results = data.get("results", []) if isinstance(data, dict) else data
    for ep in results or []:
        if ep.get("endpoint_name") == endpoint_name:
            return ep
    raise RuntimeError(f"endpoint '{endpoint_name}' not found")

def get_endpoint_key(endpoint_name: str, endpoint_key: str | None):
    if endpoint_key:
        return endpoint_key
    return get_endpoint_info(endpoint_name).get("api_key")

def get_endpoint_id(endpoint_name: str):
    return get_endpoint_info(endpoint_name).get("id")

def get_route(endpoint_name: str, endpoint_key: str, cost: float):
    payload = {"endpoint": endpoint_name, "api_key": endpoint_key, "cost": cost}
    r = requests.post(f"{VAST_RUN}/route/", json=payload, timeout=30)
    data = r.json()
    if "url" not in data:
        raise RuntimeError(f"route failed: {data}")
    return data

def get_endpoint_workers(endpoint_name: str, endpoint_key: str):
    eid = get_endpoint_id(endpoint_name)
    r = requests.post(
        f"{VAST_RUN}/get_endpoint_workers/",
        json={"id": eid, "api_key": endpoint_key},
        timeout=30,
        headers={"Content-Type": "application/json"},
    )
    data = r.json()
    if isinstance(data, dict) and "workers" in data:
        return data["workers"]
    return data if isinstance(data, list) else []

def wrap_auth(route: dict, payload: dict):
    auth = {k: route[k] for k in ["signature", "cost", "endpoint", "reqnum", "url", "request_idx"] if k in route}
    return {"auth_data": auth, "payload": payload}

async def test_health(url, endpoint_name=None, endpoint_key=None):
    print("\n=== Health ===")
    if endpoint_name:
        try:
            workers = await asyncio.to_thread(get_endpoint_workers, endpoint_name, endpoint_key)
            ok = len(workers) > 0
            print(f"  [{'PASS' if ok else 'FAIL'}] workers={len(workers)}")
            return ok
        except Exception as e:
            print(f"  [FAIL] {e}")
            return False

    async with aiohttp.ClientSession() as s:
        t0 = time.perf_counter()
        async with s.get(f"{url}/health") as r:
            dt = (time.perf_counter()-t0)*1000
            d = await r.json()
            ok = r.status == 200
            print(f"  [{'PASS' if ok else 'FAIL'}] {r.status} {dt:.0f}ms {json.dumps(d)}")
            return ok

async def test_http_batch(url, endpoint_name=None, endpoint_key=None, route_cost=10.0):
    print("\n=== HTTP Batch ===")
    _, wav = generate_test_wav(3.0)
    if endpoint_name:
        audio_b64 = base64.b64encode(wav).decode("ascii")
        async with aiohttp.ClientSession() as s:
            t0 = time.perf_counter()
            try:
                route = await asyncio.to_thread(get_route, endpoint_name, endpoint_key, route_cost)
                worker_url = route["url"].rstrip("/")
                payload = {"audio_base64": audio_b64, "language": "hi"}
                req = wrap_auth(route, payload)
                async with s.post(f"{worker_url}/v1/audio/transcriptions", json=req, timeout=aiohttp.ClientTimeout(total=120)) as r:
                    dt = (time.perf_counter()-t0)*1000
                    if r.status == 200:
                        d = await r.json()
                        print(f"  [PASS] {dt:.0f}ms text='{d.get('text','')[:80]}' proc={d.get('processing_time',0)}s")
                        return True
                    # Fallback for older images
                    if r.status in (404, 422):
                        route = await asyncio.to_thread(get_route, endpoint_name, endpoint_key, route_cost)
                        worker_url = route["url"].rstrip("/")
                        req = wrap_auth(route, payload)
                        async with s.post(f"{worker_url}/v1/audio/transcriptions/json", json=req, timeout=aiohttp.ClientTimeout(total=120)) as r2:
                            dt = (time.perf_counter()-t0)*1000
                            if r2.status == 200:
                                d = await r2.json()
                                print(f"  [PASS] {dt:.0f}ms text='{d.get('text','')[:80]}' proc={d.get('processing_time',0)}s (json)")
                                return True
                            print(f"  [FAIL] {r2.status} {(await r2.text())[:200]}")
                            return False
                    print(f"  [FAIL] {r.status} {(await r.text())[:200]}")
                    return False
            except Exception as e:
                print(f"  [FAIL] {e}"); return False

    async with aiohttp.ClientSession() as s:
        form = aiohttp.FormData()
        form.add_field('file', wav, filename='test.wav', content_type='audio/wav')
        form.add_field('language', 'hi')
        t0 = time.perf_counter()
        try:
            async with s.post(f"{url}/v1/audio/transcriptions", data=form, timeout=aiohttp.ClientTimeout(total=60)) as r:
                dt = (time.perf_counter()-t0)*1000
                if r.status == 200:
                    d = await r.json()
                    print(f"  [PASS] {dt:.0f}ms text='{d.get('text','')[:80]}' proc={d.get('processing_time',0)}s")
                    return True
                print(f"  [FAIL] {r.status} {(await r.text())[:200]}")
                return False
        except Exception as e:
            print(f"  [FAIL] {e}"); return False

async def test_http_stream(url, endpoint_name=None, endpoint_key=None, route_cost=10.0):
    print("\n=== HTTP Stream (SSE) ===")
    _, wav = generate_test_wav(3.0)
    if endpoint_name:
        audio_b64 = base64.b64encode(wav).decode("ascii")
        async with aiohttp.ClientSession() as s:
            t0 = time.perf_counter()
            try:
                route = await asyncio.to_thread(get_route, endpoint_name, endpoint_key, route_cost)
                worker_url = route["url"].rstrip("/")
                payload = {"audio_base64": audio_b64, "language": "hi", "chunk_duration_ms": 160}
                req = wrap_auth(route, payload)
                async with s.post(f"{worker_url}/v1/audio/transcriptions/stream", json=req, timeout=aiohttp.ClientTimeout(total=120)) as r:
                    if r.status != 200:
                        print(f"  [FAIL] {r.status}")
                        return False
                    events = []
                    async for line in r.content:
                        l = line.decode().strip()
                        if l.startswith("data: "):
                            d = json.loads(l[6:])
                            events.append(d)
                            if d.get("done"):
                                break
                    dt = (time.perf_counter()-t0)*1000
                    print(f"  [PASS] {len(events)} events {dt:.0f}ms")
                    if events:
                        print(f"  Final: '{events[-1].get('text','')[:80]}'")
                    return True
            except Exception as e:
                print(f"  [FAIL] {e}"); return False

    async with aiohttp.ClientSession() as s:
        form = aiohttp.FormData()
        form.add_field('file', wav, filename='test.wav', content_type='audio/wav')
        t0 = time.perf_counter()
        try:
            async with s.post(f"{url}/v1/audio/transcriptions/stream", data=form, timeout=aiohttp.ClientTimeout(total=60)) as r:
                if r.status != 200:
                    print(f"  [FAIL] {r.status}"); return False
                events = []
                async for line in r.content:
                    l = line.decode().strip()
                    if l.startswith("data: "):
                        d = json.loads(l[6:])
                        events.append(d)
                        if d.get("done"): break
                dt = (time.perf_counter()-t0)*1000
                print(f"  [PASS] {len(events)} events {dt:.0f}ms")
                if events: print(f"  Final: '{events[-1].get('text','')[:80]}'")
                return True
        except Exception as e:
            print(f"  [FAIL] {e}"); return False

async def test_websocket(url, endpoint_name=None, endpoint_key=None, route_cost=10.0, ws_port=None):
    print("\n=== WebSocket ===")
    if endpoint_name:
        if not ws_port:
            print("  [SKIP] ws_port not provided for endpoint mode")
            return True
        try:
            route = await asyncio.to_thread(get_route, endpoint_name, endpoint_key, route_cost)
            parsed = urlparse(route["url"])
            scheme = "wss" if parsed.scheme == "https" else "ws"
            host = parsed.hostname
            ws_url = f"{scheme}://{host}:{ws_port}/ws"
        except Exception as e:
            print(f"  [FAIL] {e}")
            return False
    else:
        ws_url = url.replace("http://","ws://").replace("https://","wss://") + "/ws"
    pcm, _ = generate_test_wav(3.0)
    try:
        async with aiohttp.ClientSession() as s:
            t0 = time.perf_counter()
            async with s.ws_connect(ws_url, timeout=30) as ws:
                ct = (time.perf_counter()-t0)*1000
                print(f"  Connected {ct:.0f}ms")
                ready = await asyncio.wait_for(ws.receive(), timeout=10)
                if ready.type == aiohttp.WSMsgType.TEXT:
                    print(f"  Ready: {ready.data}")
                chunk_size = int(0.160*16000*2)
                ft = None; msgs = []
                async def send():
                    for i in range(0, len(pcm), chunk_size):
                        await ws.send_bytes(pcm[i:i+chunk_size])
                        await asyncio.sleep(0.160)
                    await ws.send_str("END")
                async def recv():
                    nonlocal ft
                    while True:
                        m = await asyncio.wait_for(ws.receive(), timeout=15)
                        if m.type == aiohttp.WSMsgType.TEXT:
                            if m.data == "END": break
                            d = json.loads(m.data)
                            if ft is None and d.get("text"): ft = time.perf_counter()
                            msgs.append(d)
                        elif m.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSING): break
                await asyncio.gather(send(), recv())
                dt = (time.perf_counter()-t0)*1000
                ft_ms = (ft-t0)*1000 if ft else -1
                print(f"  [PASS] {len(msgs)} msgs {dt:.0f}ms first_token={ft_ms:.0f}ms")
                if msgs:
                    for m in reversed(msgs):
                        if m.get("text"): print(f"  Last: '{m['text'][:80]}'"); break
                return True
    except Exception as e:
        print(f"  [FAIL] {e}"); return False

async def main():
    p = argparse.ArgumentParser()
    p.add_argument("--url", default=None, help="Direct server URL (http://ip:port)")
    p.add_argument("--endpoint", default=None, help="Vast endpoint name (uses /route + auth)")
    p.add_argument("--endpoint-key", default=None, help="Endpoint API key (optional if VAST_KEY set)")
    p.add_argument("--route-cost", type=float, default=10.0)
    p.add_argument("--ws-port", type=int, default=None, help="Direct WS port on worker (if exposed)")
    p.add_argument("--http-only", action="store_true")
    p.add_argument("--ws-only", action="store_true")
    a = p.parse_args()
    if not a.url and not a.endpoint:
        print("ERROR: provide --url or --endpoint")
        sys.exit(2)
    url = a.url.rstrip("/") if a.url else ""


Output:
Chunk ID: bc2d9a
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 422
Output:
#!/bin/bash
set -e

echo "=== Hindi ASR Production Server Starting ==="
echo "GPU: $(nvidia-smi --query-gpu=name,memory.total --format=csv,noheader 2>/dev/null || echo unknown)"

export MODEL_PATH="${MODEL_PATH:-/model/final_model.nemo}"
export MAX_ACTIVE_STREAMS="${MAX_ACTIVE_STREAMS:-100}"
export ENABLE_VAD="${ENABLE_VAD:-true}"
export MODEL_LOG="${MODEL_LOG:-/var/log/model/server.log}"
export SERVER_PORT=18000
export WORKER_PORT=8000
export VAST_TCP_PORT_8000="${VAST_TCP_PORT_8000:-8000}"

mkdir -p /var/log/model /workspace

if [ ! -f "$MODEL_PATH" ]; then
    FOUND=$(find /model -name "final_model.nemo" -type f 2>/dev/null | head -1)
    if [ -n "$FOUND" ]; then
        export MODEL_PATH="$FOUND"
        echo "Found model at: $MODEL_PATH"
    fi
fi

# === Ensure PyWorker deps are installed ===
if ! python3 - <<'PY'
try:
    import vastai  # noqa: F401
    import dateutil  # noqa: F401
    print("PyWorker deps already installed")
except Exception:
    raise SystemExit(1)
PY
then
    echo "Installing PyWorker deps..."
    pip install vastai_sdk 2>&1 | tail -3
    pip install "python-dateutil>=2.8.2" 2>&1 | tail -2
fi

# === Start model server (background) ===
echo "Starting ASR model server on port $SERVER_PORT..."
cd /app
python3 server.py 2>&1 | tee -a "$MODEL_LOG" &
SERVER_PID=$!

# === Start PyWorker (background) ===
echo "Starting PyWorker on port $WORKER_PORT..."
cd /app
python3 worker.py 2>&1 | tee -a /workspace/pyworker.log &
PYWORKER_PID=$!

echo "Server PID=$SERVER_PID, PyWorker PID=$PYWORKER_PID"
wait -n $SERVER_PID $PYWORKER_PID 2>/dev/null || true
echo "A process exited. Shutting down..."
kill $SERVER_PID $PYWORKER_PID 2>/dev/null || true
wait


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Planning deeper file inspection**


I’ve confirmed metrics exist but they’re still weak for HTTP/SSE (GPU-time percentiles are tied to WS batching), so I’ll now inspect the remaining server code and existing benchmark artifacts to identify metric gaps and then patch instrumentation before re-running long sweeps.


sed -n '280,620p' production/test_smoke.py


sed -n '1,260p' production/requirements.txt


Output:
Chunk ID: 85fd68
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 4157
Output:
                                    })
                                    yield f"data: {event_data}\n\n"
                    except Exception:
                        break

                # Final event with complete transcription
                yield f"data: {json.dumps({'text': full_text, 'is_final': True, 'done': True})}\n\n"

            finally:
                # Cleanup
                if stream_id in state.pipeline._state_pool:
                    del state.pipeline._state_pool[stream_id]
                if stream_id in state.pipeline._streaming_request_generator.streams:
                    state.pipeline._streaming_request_generator.streams.pop(stream_id, None)

        state.metrics["http_requests_ok"] += 1
        return StreamingResponse(
            sse_generator(),
            media_type="text/event-stream",
            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
        )

    except HTTPException:
        raise
    except Exception as e:
        state.metrics["http_requests_err"] += 1
        logger.error(f"HTTP stream error: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail=str(e))

# ============================================================
# WebSocket Streaming
# ============================================================

@app.websocket("/ws")
async def websocket_streaming(ws: WebSocket):
    """
    Production WebSocket endpoint for real-time streaming ASR.

    Protocol:
      Client -> Server: binary audio chunks (raw PCM 16-bit 16kHz mono)
      Client -> Server: text "END" to signal end of audio
      Server -> Client: JSON {"text": "...", "is_final": bool, "timestamp": float}
      Server -> Client: text "END" to confirm stream complete

    Backpressure: Rejects with 1013 when at capacity.
    """
    state.metrics["ws_connections_total"] += 1

    # Backpressure: fast-fail when overloaded
    active = len(state.client_queues)
    if active >= MAX_ACTIVE_STREAMS:
        state.metrics["ws_connections_rejected"] += 1
        await ws.accept()
        await ws.close(code=1013, reason=f"Overloaded: {active}/{MAX_ACTIVE_STREAMS}")
        return

    await ws.accept()
    state.metrics["ws_connections_active"] += 1

    stream_id = None
    try:
        # Start inference loop if needed
        start_inference_loop_if_needed()

        # Open streaming session
        stream_id = state.pipeline.open_streaming_session()
        transcription_queue = await register_client(stream_id)
        elapsed = time.perf_counter() - state.start_time
        logger.info(f"[+{elapsed:.1f}s] WS client {stream_id} connected ({len(state.client_queues)} total)")

    except Exception as e:
        logger.error(f"WS connection setup error: {e}", exc_info=True)
        state.metrics["ws_connections_active"] -= 1
        if stream_id is not None:
            await unregister_client(stream_id)
        if ws.application_state is WebSocketState.CONNECTED:
            await ws.close()
        return

    # Send ready message
    await ws.send_text(json.dumps({"type": "Ready", "id": stream_id}))

    async def recv_loop():
        """Receive audio chunks from client, feed to pipeline"""
        att_ctx = state.cfg.streaming.att_context_size[1] + 1
        frame_threshold = att_ctx * 1280 * 2  # bytes
        audio_buffer = b""

        try:
            while True:
                message = await ws.receive()

                # Text message = control signal
                if "text" in message:
                    text_msg = message["text"]
                    if text_msg == "END":
                        await transcription_queue.put({"type": "marker"})
                        break

                # Binary message = audio data
                if "bytes" in message:
                    raw_bytes = message["bytes"]
                    if isinstance(raw_bytes, str):
                        raw_bytes = raw_bytes.encode("latin-1")
                    audio_buffer += raw_bytes

                    # Feed to pipeline when buffer exceeds frame threshold
                    if len(audio_buffer) >= frame_threshold:
                        # VAD gate: skip silence before GPU
                        if state.vad is not None and not state.vad.is_speech(audio_buffer, stream_id):
                            state.metrics["vad_filtered_chunks"] += 1
                            audio_buffer = b""
                            continue

                        state.pipeline.append_streaming_audio(stream_id, audio_buffer)
                        audio_buffer = b""

                        # Signal inference loop
                        state._batch_ready_event.set()
                        await asyncio.sleep(0)

        except WebSocketDisconnect:
            pass
        except Exception as e:
            logger.error(f"WS recv error stream {stream_id}: {e}")
        finally:
            # Flush remaining buffer
            if audio_buffer:
                try:
                    state.pipeline.append_streaming_audio(stream_id, audio_buffer)
                except Exception:
                    pass

            # Mark stream as ended
            try:
                state.pipeline._streaming_request_generator.streams[stream_id].mark_end()
            except Exception:
                pass

            state._batch_ready_event.set()
            await asyncio.sleep(0.1)
            await transcription_queue.put(None)  # Shutdown signal

    async def send_loop():
        """Send transcription results to client"""
        step_num = 0
        try:
            while True:
                output_dict = await transcription_queue.get()
                if output_dict is None:
                    break

                if output_dict.get("type") == "marker":
                    await ws.send_text("END")
                    break

                json_str = output_dict.get("json_str")
                if json_str:
                    if step_num == 0:
                        elapsed = time.perf_counter() - state.start_time
                        logger.info(f"[+{elapsed:.1f}s] Stream {stream_id}: first token sent")
                    await ws.send_text(json_str)
                    step_num += 1

        except Exception as e:
            logger.error(f"WS send error stream {stream_id}: {e}")

    tasks = []
    try:
        tasks = [
            asyncio.create_task(recv_loop()),
            asyncio.create_task(send_loop()),
        ]
        await asyncio.gather(*tasks, return_exceptions=True)
    except Exception as e:
        logger.error(f"WS error stream {stream_id}: {e}")
    finally:
        state.metrics["ws_connections_active"] -= 1
        await unregister_client(stream_id)
        if ws.application_state is WebSocketState.CONNECTED:
            await ws.close()
        for task in tasks:
            if not task.done():
                task.cancel()
                try:
                    await task
                except asyncio.CancelledError:
                    pass

# ============================================================
# Stream Management + Inference Loop
# ============================================================

async def register_client(stream_id: int) -> asyncio.Queue:
    """Register a streaming client, return its output queue"""
    queue = asyncio.Queue()
    async with state.client_queues_lock:
        state.client_queues[stream_id] = queue
    async with state.stream_timestamps_lock:
        state.stream_audio_timestamps[stream_id] = 0.0
    return queue

async def unregister_client(stream_id: int):
    """Remove client from tracking, mark for cleanup"""
    try:
        async with state.client_queues_lock:
            state.client_queues.pop(stream_id, None)
        async with state.stream_timestamps_lock:
            state.stream_audio_timestamps.pop(stream_id, None)
        if state.vad is not None:
            state.vad.reset_stream(stream_id)
        if hasattr(state.pipeline, "_streaming_request_generator"):
            state.pipeline._streaming_request_generator.close_stream(stream_id)
        async with state.streams_cleanup_lock:
            state.streams_pending_cleanup.add(stream_id)
    except Exception as e:
        logger.error(f"Unregister error stream {stream_id}: {e}")

async def route_outputs(outputs_with_timestamps):
    """Route inference outputs to per-client queues"""
    current_time = time.time()

    for output_dict in outputs_with_timestamps:
        output = output_dict["output"]
        audio_ts = output_dict["audio_timestamp"]

        text = output.current_step_transcript if output.partial_transcript else output.final_transcript
        is_final = not bool(output.partial_transcript)

        result = {
            "text": text or "",
            "is_final": is_final,
            "timestamp": current_time,
            "audio_timestamp": audio_ts,
        }

        if len(output.final_segments) > 0:
            result["segment_text"] = output.final_segments[0].text
            result["segment_start"] = output.final_segments[0].start
            result["segment_end"] = output.final_segments[0].end

        output_dict["json_str"] = json.dumps(result)

    # Snapshot queues without holding lock during I/O
    async with state.client_queues_lock:
        queues_snapshot = dict(state.client_queues)

    for output_dict in outputs_with_timestamps:
        output = output_dict["output"]
        queue = queues_snapshot.get(output.stream_id)
        if queue:
            try:
                await queue.put(output_dict)
            except Exception as e:
                logger.error(f"Route error stream {output.stream_id}: {e}")

async def cleanup_finished_streams():
    """Clean up streams fully drained from the request generator"""
    async with state.streams_cleanup_lock:
        if not state.streams_pending_cleanup:
            return
        active_ids = set(state.pipeline._streaming_request_generator.streams.keys())
        to_cleanup = state.streams_pending_cleanup - active_ids
        for sid in to_cleanup:
            try:
                if sid in state.pipeline._state_pool:
                    del state.pipeline._state_pool[sid]
                if hasattr(state.pipeline, "context_manager") and state.pipeline.context_manager is not None:
                    try:
                        state.pipeline.context_manager.reset_slots([sid], [True])
                    except Exception:
                        pass
                state.streams_pending_cleanup.discard(sid)
            except Exception as e:
                logger.error(f"Cleanup error stream {sid}: {e}")

async def centralized_inference_loop():
    """
    Production inference loop with event-driven batching.

    WHY event-driven: fires immediately when data arrives (high load) or
    after MAX_BATCH_WAIT_MS timeout (low load). 6x better than fixed 300ms sleep.
    """
    state.inference_running = True
    max_wait = MAX_BATCH_WAIT_MS / 1000.0
    metrics_report_interval = 100

    logger.info(f"Inference loop started (max_wait={MAX_BATCH_WAIT_MS}ms)")

    while state.inference_running:
        try:
            active = len(state.client_queues)

            # Idle if no clients
            if active == 0:
                try:
                    await asyncio.wait_for(state._batch_ready_event.wait(), timeout=0.1)
                    state._batch_ready_event.clear()
                except asyncio.TimeoutError:
                    pass
                continue

            # Event-driven wait
            cycle_start = time.perf_counter()
            try:
                await asyncio.wait_for(state._batch_ready_event.wait(), timeout=max_wait)
            except asyncio.TimeoutError:
                pass
            state._batch_ready_event.clear()

            # Process ALL ready frames (exhaustive)
            all_outputs = []
            async with state.stream_timestamps_lock:
                timestamps = dict(state.stream_audio_timestamps)

            batches_this_cycle = 0
            while True:
                try:
                    batch_start = time.perf_counter()
                    requests, outputs = await state.pipeline.process_streaming_batch()
                    gpu_ms = (time.perf_counter() - batch_start) * 1000

                    if not outputs:
                        break

                    batches_this_cycle += 1
                    state.metrics["total_batches"] += 1
                    state.metrics["total_outputs"] += len(outputs)
                    state.metrics["total_gpu_ms"] += gpu_ms
                    state.metrics["batch_sizes"].append(len(outputs))
                    state.metrics["gpu_times_ms"].append(gpu_ms)

                    # Update timestamps
                    for req in requests:
                        if hasattr(req, "length"):
                            sid = req.stream_id
                            timestamps[sid] = timestamps.get(sid, 0.0) + req.length / 16000.0

                    for output in outputs:
                        all_outputs.append({
                            "output": output,
                            "audio_timestamp": timestamps.get(output.stream_id, 0.0),
                        })

                    await asyncio.sleep(0)

                except Exception as e:
                    if type(e).__name__ == "NotEnoughDataException":
                        if batches_this_cycle == 0:
                            state.metrics["empty_cycles"] += 1
                        break
                    raise

            if all_outputs:
                async with state.stream_timestamps_lock:
                    for sid, ts in timestamps.items():
                        if sid in state.stream_audio_timestamps:
                            state.stream_audio_timestamps[sid] = ts

                route_start = time.perf_counter()
                await route_outputs(all_outputs)
                state.metrics["total_route_ms"] += (time.perf_counter() - route_start) * 1000

                await cleanup_finished_streams()

                cycle_ms = (time.perf_counter() - cycle_start) * 1000
                state.metrics["cycle_times_ms"].append(cycle_ms)

                # Periodic metrics log
                if state.metrics["total_batches"] % metrics_report_interval == 0:
                    _log_metrics(active)

        except Exception as e:
            logger.error(f"Inference loop error: {e}", exc_info=True)
            await asyncio.sleep(0.1)

    logger.warning("Inference loop stopped")

def _log_metrics(active_clients: int):
    """Structured metrics log"""
    m = state.metrics

    def pct(arr, p):
        if not arr:
            return 0
        s = sorted(arr)
        return s[min(int(len(s) * p / 100), len(s) - 1)]

    try:
        vram_used = torch.cuda.memory_allocated() / 1e9
        vram_total = torch.cuda.get_device_properties(0).total_memory / 1e9
    except Exception:
        vram_used = vram_total = 0

    elapsed = time.perf_counter() - state.start_time
    logger.info(
        f"[+{elapsed:.0f}s] METRICS | streams={active_clients} "
        f"batches={m['total_batches']} outputs={m['total_outputs']} "
        f"GPU: p50={pct(m['gpu_times_ms'], 50):.1f}ms p99={pct(m['gpu_times_ms'], 99):.1f}ms | "
        f"VRAM: {vram_used:.1f}/{vram_total:.0f}GB | "
        f"vad_skip={m['vad_filtered_chunks']}"
    )

def start_inference_loop_if_needed():
    """Start centralized inference loop on first client connection"""
    if state.inference_task is None:
        logger.info("Starting inference loop (first client)")
        state.inference_task = asyncio.create_task(centralized_inference_loop())
    elif state.inference_task.done():
        logger.warning("Inference task was done, restarting")
        state.inference_task = asyncio.create_task(centralized_inference_loop())

# ============================================================
# Main
# ============================================================

if __name__ == "__main__":
    logger.info(f"Starting ASR server on port {SERVER_PORT}")
    logger.info(f"  Model: {MODEL_PATH}")
    logger.info(f"  Max streams: {MAX_ACTIVE_STREAMS}")
    logger.info(f"  Batch wait: {MAX_BATCH_WAIT_MS}ms")
    logger.info(f"  VAD: {'enabled' if ENABLE_VAD else 'disabled'}")

    uvicorn.run(
        app,
        host="0.0.0.0",
        port=SERVER_PORT,
        log_level="info",
        access_log=False,  # Reduce noise, we have structured logging
        timeout_keep_alive=300,  # 5min for long WebSocket sessions
    )


Output:
Chunk ID: 4e4fa0
Wall time: 0.0503 seconds
Process exited with code 0
Original token count: 211
Output:
    url = a.url.rstrip("/") if a.url else ""
    endpoint_key = get_endpoint_key(a.endpoint, a.endpoint_key) if a.endpoint else None
    results = {"health": await test_health(url, a.endpoint, endpoint_key)}
    if not a.ws_only:
        results["http_batch"] = await test_http_batch(url, a.endpoint, endpoint_key, a.route_cost)
        results["http_stream"] = await test_http_stream(url, a.endpoint, endpoint_key, a.route_cost)
    if not a.http_only:
        results["websocket"] = await test_websocket(url, a.endpoint, endpoint_key, a.route_cost, a.ws_port)
    print("\n=== SUMMARY ===")
    ok = all(results.values())
    for k,v in results.items(): print(f"  {k:20s} [{'PASS' if v else 'FAIL'}]")
    print(f"\n{'All passed!' if ok else 'FAILURES detected'}")
    sys.exit(0 if ok else 1)

if __name__ == "__main__": asyncio.run(main())


Output:
Chunk ID: ed3f50
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 9
Output:
vastai_sdk
python-dateutil>=2.8.2


Output:
Chunk ID: bf347a
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 563
Output:
total 4576
-rw-rw-r-- 1 ubuntu ubuntu  15995 Feb  9 20:27 stress_results_20260209_202718.json
-rw-rw-r-- 1 ubuntu ubuntu    724 Feb  9 20:27 stress_results_20260209_202718.csv
-rw-rw-r-- 1 ubuntu ubuntu  15974 Feb  9 20:24 stress_results_20260209_202433.json
-rw-rw-r-- 1 ubuntu ubuntu    736 Feb  9 20:24 stress_results_20260209_202433.csv
-rw-rw-r-- 1 ubuntu ubuntu  13119 Feb  9 20:14 stress_results_20260209_201434.json
-rw-rw-r-- 1 ubuntu ubuntu    725 Feb  9 20:14 stress_results_20260209_201434.csv
-rw-rw-r-- 1 ubuntu ubuntu   8403 Feb  9 19:57 stress_results_20260209_195730.json
-rw-rw-r-- 1 ubuntu ubuntu    543 Feb  9 19:57 stress_results_20260209_195730.csv
-rw-rw-r-- 1 ubuntu ubuntu   8418 Feb  9 19:56 stress_results_20260209_195631.json
-rw-rw-r-- 1 ubuntu ubuntu    552 Feb  9 19:56 stress_results_20260209_195631.csv
-rw-rw-r-- 1 ubuntu ubuntu  10691 Feb  9 07:01 capacity_sweep_a100_80gb.json
-rw-rw-r-- 1 ubuntu ubuntu    330 Feb  9 07:01 capacity_c200_s60_health.csv
-rw-rw-r-- 1 ubuntu ubuntu  77985 Feb  9 07:01 capacity_c200_s60_streams.csv
-rw-rw-r-- 1 ubuntu ubuntu    452 Feb  9 07:01 capacity_c200_s60.json
-rw-rw-r-- 1 ubuntu ubuntu    329 Feb  9 07:00 capacity_c175_s60_health.csv
-rw-rw-r-- 1 ubuntu ubuntu  71910 Feb  9 07:00 capacity_c175_s60_streams.csv
-rw-rw-r-- 1 ubuntu ubuntu    704 Feb  9 07:00 capacity_c175_s60.json
-rw-rw-r-- 1 ubuntu ubuntu    305 Feb  9 06:59 capacity_c150_s60_health.csv
-rw-rw-r-- 1 ubuntu ubuntu  93177 Feb  9 06:59 capacity_c150_s60_streams.csv
-rw-rw-r-- 1 ubuntu ubuntu    706 Feb  9 06:59 capacity_c150_s60.json
-rw-rw-r-- 1 ubuntu ubuntu    327 Feb  9 06:58 capacity_c140_s60_health.csv
-rw-rw-r-- 1 ubuntu ubuntu  98541 Feb  9 06:58 capacity_c140_s60_streams.csv
-rw-rw-r-- 1 ubuntu ubuntu    692 Feb  9 06:58 capacity_c140_s60.json
-rw-rw-r-- 1 ubuntu ubuntu    303 Feb  9 06:58 capacity_c130_s60_health.csv
-rw-rw-r-- 1 ubuntu ubuntu  88983 Feb  9 06:58 capacity_c130_s60_streams.csv
-rw-rw-r-- 1 ubuntu ubuntu    701 Feb  9 06:58 capacity_c130_s60.json
-rw-rw-r-- 1 ubuntu ubuntu    327 Feb  9 06:57 capacity_c120_s60_health.csv
-rw-rw-r-- 1 ubuntu ubuntu  83316 Feb  9 06:57 capacity_c120_s60_streams.csv
-rw-rw-r-- 1 ubuntu ubuntu    699 Feb  9 06:57 capacity_c120_s60.json


Summary: auto


**Planning WER testing and benchmarking**


I confirmed the main gap: HTTP/SSE currently report latency and VRAM, but not per-request GPU timing or quality metrics. I’ll now locate existing WER/evaluation assets in the repo so we can integrate WER checks into the same stress workflow instead of creating a disconnected test.


rg -n "WER|wer|CER|cer|word error|transcription|ground truth|ground_truth|reference" -S --glob '!venv/**'


find . -maxdepth 3 -type f \( -name '*.wav' -o -name '*.mp3' -o -name '*.flac' -o -name '*.jsonl' -o -name '*manifest*' -o -name '*transcript*' -o -name '*ground*' \) | sed 's#^./##' | head -n 200


ls -la vast_docs && rg -n "serverless|endpoint|workergroup|autoscaling|util|target|cold|queue|route|metrics|geolocation|region" vast_docs -S