Output:
Chunk ID: 915c8d
Wall time: 0.0516 seconds
Process exited with code 0
Original token count: 5858
Output:
   360	    # Initialize VAD
   361	    if ENABLE_VAD:
   362	        from vad import SileroVAD
   363	        state.vad = SileroVAD(threshold=VAD_THRESHOLD, sample_rate=16000)
   364	        logger.info(f"VAD enabled (threshold={VAD_THRESHOLD})")
   365	    else:
   366	        logger.info("VAD disabled")
   367
   368	    state.start_time = time.perf_counter()
   369	    state.model_loaded = True
   370
   371	    # CRITICAL: This log line triggers Vast.ai PyWorker readiness detection
   372	    logger.info("Application startup complete.")
   373	    print("Application startup complete.", flush=True)
   374
   375
   376	async def transcribe_pcm_bytes_streaming(pcm_bytes: bytes) -> str:
   377	    """
   378	    Transcribe PCM bytes using the streaming pipeline (no .run() required).
   379	    WHY: CacheAwareRNNTStreamingPipeline doesn't implement .run().
   380	    """
   381	    from nemo.collections.asr.inference.streaming.framing.request_options import ASRRequestOptions
   382
   383	    options = ASRRequestOptions()
   384	    options.asr_output_granularity = "word"
   385	    stream_id = state.pipeline.open_streaming_session(options=options)
   386
   387	    # Use pipeline's native chunk size if available; fallback to 80ms
   388	    frame_secs = getattr(state.pipeline, "chunk_size_in_secs", 0.08) or 0.08
   389	    frame_bytes = max(2560, int(frame_secs * 16000 * 2))  # 16kHz, 16-bit
   390
   391	    full_segments: list[str] = []
   392	    last_partial = ""
   393
   394	    try:
   395	        # Feed audio in frame-sized chunks
   396	        for i in range(0, len(pcm_bytes), frame_bytes):
   397	            chunk = pcm_bytes[i:i + frame_bytes]
   398	            state.pipeline.append_streaming_audio(stream_id, chunk)
   399	            try:
   400	                requests, outputs = await state.pipeline.process_streaming_batch()
   401	            except Exception as e:
   402	                if type(e).__name__ == "NotEnoughDataException":
   403	                    outputs = []
   404	                else:
   405	                    raise
   406	            for output in outputs:
   407	                if output.stream_id != stream_id:
   408	                    continue
   409	                if output.final_transcript:
   410	                    full_segments.append(output.final_transcript.strip())
   411	                    # Final transcript commits buffered tokens; clear stale partial.
   412	                    last_partial = ""
   413	                if output.partial_transcript:
   414	                    last_partial = output.partial_transcript.strip()
   415
   416	        # Close and drain remaining outputs
   417	        state.pipeline.close_streaming_session(stream_id)
   418	        for _ in range(10):
   419	            try:
   420	                requests, outputs = await state.pipeline.process_streaming_batch()
   421	            except Exception as e:
   422	                if type(e).__name__ == "NotEnoughDataException":
   423	                    outputs = []
   424	                else:
   425	                    break
   426	            if not outputs:
   427	                break
   428	            for output in outputs:
   429	                if output.stream_id != stream_id:
   430	                    continue
   431	                if output.final_transcript:
   432	                    full_segments.append(output.final_transcript.strip())
   433	                    last_partial = ""
   434	                if output.partial_transcript:
   435	                    last_partial = output.partial_transcript.strip()
   436
   437	    finally:
   438	        # Cleanup stream state
   439	        if stream_id in state.pipeline._state_pool:
   440	            del state.pipeline._state_pool[stream_id]
   441	        if stream_id in state.pipeline._streaming_request_generator.streams:
   442	            state.pipeline._streaming_request_generator.streams.pop(stream_id, None)
   443
   444	    # Keep tail partial to avoid truncating long utterances with no final EOU flush.
   445	    combined = " ".join([x for x in [*full_segments, last_partial] if x]).strip()
   446	    final_only = " ".join([x for x in full_segments if x]).strip()
   447	    if len(combined) >= len(final_only):
   448	        return combined
   449	    return final_only or last_partial
   450
   451
   452	# ============================================================
   453	# HTTP Endpoints
   454	# ============================================================
   455
   456	async def _parse_audio_request(request: Request):
   457	    """
   458	    Parse audio payload from either JSON (audio_base64) or multipart form-data (file).
   459	    Returns (audio_bytes, language, response_format, chunk_duration_ms).
   460	    """
   461	    content_type = request.headers.get("content-type", "")
   462
   463	    if content_type.startswith("application/json"):
   464	        body = await request.json()
   465	        audio_b64 = (
   466	            body.get("audio_base64")
   467	            or body.get("audio_b64")
   468	            or body.get("audio")
   469	        )
   470	        if not audio_b64:
   471	            raise HTTPException(status_code=400, detail="audio_base64 required")
   472	        try:
   473	            audio_data = base64.b64decode(audio_b64)
   474	        except Exception as e:
   475	            raise HTTPException(status_code=400, detail=f"invalid audio_base64: {e}")
   476	        language = body.get("language", "hi")
   477	        response_format = body.get("response_format", "json")
   478	        chunk_duration_ms = int(body.get("chunk_duration_ms", 160) or 160)
   479	        return audio_data, language, response_format, chunk_duration_ms
   480
   481	    # Fallback: multipart form-data
   482	    form = await request.form()
   483	    file_obj = form.get("file")
   484	    if file_obj is None:
   485	        raise HTTPException(status_code=400, detail="file required")
   486
   487	    if hasattr(file_obj, "read"):
   488	        audio_data = await file_obj.read()
   489	    else:
   490	        audio_data = file_obj
   491
   492	    language = form.get("language", "hi")
   493	    response_format = form.get("response_format", "json")
   494	    chunk_duration_ms = int(form.get("chunk_duration_ms", 160) or 160)
   495	    return audio_data, language, response_format, chunk_duration_ms
   496
   497	@app.on_event("startup")
   498	async def startup():
   499	    """Load model on server startup"""
   500	    await load_model()
   501
   502
   503	@app.get("/health")
   504	async def health_check():
   505	    """
   506	    Health check for load balancers and PyWorker.
   507	    Returns 503 when overloaded so /route/ skips this worker.
   508	    """
   509	    try:
   510	        vram_used = torch.cuda.memory_allocated() / 1e9
   511	        vram_total = torch.cuda.get_device_properties(0).total_memory / 1e9
   512	        gpu_name = torch.cuda.get_device_name(0)
   513	    except Exception:
   514	        vram_used = vram_total = 0
   515	        gpu_name = "unknown"
   516
   517	    active = len(state.client_queues)
   518	    status = "healthy" if active < MAX_ACTIVE_STREAMS else "overloaded"
   519	    uptime = round(time.perf_counter() - state.start_time, 0) if state.start_time else 0
   520
   521	    return JSONResponse(
   522	        status_code=200 if status == "healthy" else 503,
   523	        content={
   524	            "status": status,
   525	            "model_loaded": state.model_loaded,
   526	            "active_streams": active,
   527	            "max_streams": MAX_ACTIVE_STREAMS,
   528	            "gpu": gpu_name,
   529	            "vram_used_gb": round(vram_used, 2),
   530	            "vram_total_gb": round(vram_total, 0),
   531	            "inference_running": state.inference_running,
   532	            "uptime_s": uptime,
   533	        },
   534	    )
   535
   536
   537	def _build_metrics_payload():
   538	    """Shared metrics payload for GET/POST /metrics."""
   539	    m = state.metrics
   540
   541	    def pct(arr, p):
   542	        if not arr:
   543	            return 0
   544	        s = sorted(arr)
   545	        idx = min(int(len(s) * p / 100), len(s) - 1)
   546	        return round(s[idx], 2)
   547
   548	    cuda_available = False
   549	    device_count = 0
   550	    gpu_name = "unknown"
   551	    vram_used = vram_total = vram_reserved = 0
   552	    try:
   553	        cuda_available = torch.cuda.is_available()
   554	        device_count = torch.cuda.device_count() if cuda_available else 0
   555	        if cuda_available and device_count > 0:
   556	            gpu_name = torch.cuda.get_device_name(0)
   557	            vram_total = torch.cuda.get_device_properties(0).total_memory / 1e9
   558	            vram_used = torch.cuda.memory_allocated() / 1e9
   559	            vram_reserved = torch.cuda.memory_reserved() / 1e9
   560	    except Exception:
   561	        pass
   562
   563	    return {
   564	        "active_streams": len(state.client_queues),
   565	        "max_streams": MAX_ACTIVE_STREAMS,
   566	        "inference": {
   567	            "total_batches": m["total_batches"],
   568	            "total_outputs": m["total_outputs"],
   569	            "empty_cycles": m["empty_cycles"],
   570	            "vad_filtered_chunks": m["vad_filtered_chunks"],
   571	            "gpu_ms_p50": pct(m["gpu_times_ms"], 50),
   572	            "gpu_ms_p99": pct(m["gpu_times_ms"], 99),
   573	            "cycle_ms_p50": pct(m["cycle_times_ms"], 50),
   574	            "cycle_ms_p99": pct(m["cycle_times_ms"], 99),
   575	            "batch_size_avg": round(sum(m["batch_sizes"]) / max(len(m["batch_sizes"]), 1), 1),
   576	        },
   577	        "http": {
   578	            "requests_total": m["http_requests_total"],
   579	            "requests_ok": m["http_requests_ok"],
   580	            "requests_err": m["http_requests_err"],
   581	        },
   582	        "ws": {
   583	            "connections_total": m["ws_connections_total"],
   584	            "connections_active": m["ws_connections_active"],
   585	            "connections_rejected": m["ws_connections_rejected"],
   586	        },
   587	        "cuda_available": cuda_available,
   588	        "cuda_device_count": device_count,
   589	        "gpu_name": gpu_name,
   590	        "vram_used_gb": round(vram_used, 2),
   591	        "vram_reserved_gb": round(vram_reserved, 2),
   592	        "vram_total_gb": round(vram_total, 0),
   593	        "uptime_s": round(time.perf_counter() - state.start_time, 0) if state.start_time else 0,
   594	    }
   595
   596
   597	@app.get("/metrics")
   598	async def metrics_get():
   599	    """
   600	    Detailed metrics endpoint for monitoring/alerting.
   601	    Returns structured JSON with rolling-window percentiles.
   602	    """
   603	    return _build_metrics_payload()
   604
   605
   606	@app.post("/metrics")
   607	async def metrics_post(_request: Request):
   608	    """POST variant for serverless PyWorker proxy."""
   609	    return _build_metrics_payload()
   610
   611
   612	@app.post("/v1/audio/transcriptions/json")
   613	async def transcribe_json(request: Request):
   614	    """
   615	    JSON-based transcription endpoint for PyWorker benchmarks.
   616	    Accepts: {"audio_base64": "...", "language": "hi"}
   617	    WHY: PyWorker sends benchmark payloads as JSON, not multipart.
   618	    """
   619	    state.metrics["http_requests_total"] += 1
   620	    if not state.model_loaded:
   621	        state.metrics["http_requests_err"] += 1
   622	        raise HTTPException(status_code=503, detail="Model not loaded yet")
   623	    t_start = time.perf_counter()
   624	    try:
   625	        audio_data, language, _response_format, _chunk_duration_ms = await _parse_audio_request(request)
   626	        from asr_utils import preprocess_audio
   627	        pcm_bytes = preprocess_audio(audio_data, target_sample_rate=16000)
   628	        audio_duration = len(pcm_bytes) / (16000 * 2)
   629	        t_infer = time.perf_counter()
   630	        text = await transcribe_pcm_bytes_streaming(pcm_bytes)
   631	        t_done = time.perf_counter()
   632	        state.metrics["http_requests_ok"] += 1
   633	        return {
   634	            "text": text,
   635	            "duration": round(audio_duration, 2),
   636	            "processing_time": round(t_done - t_infer, 3),
   637	        }
   638	    except HTTPException:
   639	        raise
   640	    except Exception as e:
   641	        state.metrics["http_requests_err"] += 1
   642	        logger.error(f"JSON transcribe error: {e}", exc_info=True)
   643	        raise HTTPException(status_code=500, detail=str(e))
   644
   645
   646	@app.post("/v1/audio/transcriptions")
   647	async def transcribe_batch(request: Request):
   648	    """
   649	    OpenAI-compatible batch transcription endpoint.
   650	    Accepts audio file, returns full transcription.
   651
   652	    WHY HTTP batch: For non-real-time use cases (file transcription, post-processing).
   653	    Uses pipeline.run() directly — simpler, no streaming overhead.
   654	    """
   655	    state.metrics["http_requests_total"] += 1
   656
   657	    if not state.model_loaded:
   658	        state.metrics["http_requests_err"] += 1
   659	        raise HTTPException(status_code=503, detail="Model not loaded yet")
   660
   661	    t_start = time.perf_counter()
   662
   663	    try:
   664	        # Read uploaded audio (multipart or JSON base64)
   665	        audio_data, language, response_format, _chunk_duration_ms = await _parse_audio_request(request)
   666	        if not audio_data:
   667	            raise HTTPException(status_code=400, detail="Empty audio file")
   668
   669	        # Preprocess audio to 16kHz mono PCM
   670	        from asr_utils import preprocess_audio
   671	        pcm_bytes = preprocess_audio(audio_data, target_sample_rate=16000)
   672	        audio_duration = len(pcm_bytes) / (16000 * 2)  # seconds
   673
   674	        # Run streaming inference (batch)
   675	        t_infer = time.perf_counter()
   676	        text = await transcribe_pcm_bytes_streaming(pcm_bytes)
   677	        t_done = time.perf_counter()
   678
   679	        result = {
   680	            "text": text,
   681	            "language": language,
   682	            "duration": round(audio_duration, 2),
   683	            "processing_time": round(t_done - t_infer, 3),
   684	            "total_time": round(t_done - t_start, 3),
   685	        }
   686
   687	        state.metrics["http_requests_ok"] += 1
   688	        logger.info(f"HTTP batch: {audio_duration:.1f}s audio -> {t_done - t_infer:.2f}s inference")
   689	        return result
   690
   691	    except HTTPException:
   692	        raise
   693	    except Exception as e:
   694	        state.metrics["http_requests_err"] += 1
   695	        logger.error(f"HTTP batch error: {e}", exc_info=True)
   696	        raise HTTPException(status_code=500, detail=str(e))
   697
   698
   699	@app.post("/v1/audio/transcriptions/stream")
   700	async def transcribe_stream(request: Request):
   701	    """
   702	    HTTP streaming transcription via Server-Sent Events (SSE).
   703	    Client uploads audio file, gets progressive transcription events.
   704
   705	    WHY SSE: Works through HTTP proxies (including Vast.ai PyWorker).
   706	    Compatible with standard HTTP clients — no WebSocket library needed.
   707	    """
   708	    state.metrics["http_requests_total"] += 1
   709
   710	    if not state.model_loaded:
   711	        state.metrics["http_requests_err"] += 1
   712	        raise HTTPException(status_code=503, detail="Model not loaded yet")
   713
   714	    try:
   715	        audio_data, language, response_format, chunk_duration_ms = await _parse_audio_request(request)
   716	        if not audio_data:
   717	            raise HTTPException(status_code=400, detail="Empty audio file")
   718
   719	        from asr_utils import preprocess_audio
   720	        pcm_bytes = preprocess_audio(audio_data, target_sample_rate=16000)
   721
   722	        async def sse_generator():
   723	            """Stream transcription results as SSE events"""
   724	            from nemo.collections.asr.inference.streaming.framing.request_options import ASRRequestOptions
   725
   726	            options = ASRRequestOptions()
   727	            options.asr_output_granularity = "word"
   728	            stream_id = state.pipeline.open_streaming_session(options=options)
   729
   730	            chunk_size = int(chunk_duration_ms / 1000.0 * 16000 * 2)  # bytes per chunk
   731	            att_ctx = state.cfg.streaming.att_context_size[1] + 1
   732	            frame_bytes = att_ctx * 1280 * 2  # samples per frame * 2 bytes
   733
   734	            audio_buffer = b""
   735	            full_segments: list[str] = []
   736	            last_partial = ""
   737
   738	            try:
   739	                for i in range(0, len(pcm_bytes), chunk_size):
   740	                    chunk = pcm_bytes[i:i + chunk_size]
   741	                    audio_buffer += chunk
   742
   743	                    # Feed to pipeline when we have enough for a frame
   744	                    if len(audio_buffer) >= frame_bytes:
   745	                        state.pipeline.append_streaming_audio(stream_id, audio_buffer)
   746	                        audio_buffer = b""
   747
   748	                        # Process batch
   749	                        requests, outputs = await state.pipeline.process_streaming_batch()
   750	                        for output in outputs:
   751	                            if output.stream_id == stream_id:
   752	                                text = output.final_transcript or output.partial_transcript or ""
   753	                                is_final = bool(output.final_transcript)
   754	                                if text.strip():
   755	                                    if is_final:
   756	                                        full_segments.append(text.strip())
   757	                                        last_partial = ""
   758	                                    else:
   759	                                        last_partial = text.strip()
   760	                                    event_data = json.dumps({
   761	                                        "text": text,
   762	                                        "is_final": is_final,
   763	                                        "timestamp": time.time(),
   764	                                    })
   765	                                    yield f"data: {event_data}\n\n"
   766
   767	                    # Simulate real-time pacing (optional, for more realistic SSE)
   768	                    await asyncio.sleep(0)
   769
   770	                # Flush remaining audio
   771	                if audio_buffer:
   772	                    state.pipeline.append_streaming_audio(stream_id, audio_buffer)
   773
   774	                # Close stream and drain remaining outputs
   775	                state.pipeline.close_streaming_session(stream_id)
   776	                for _ in range(10):
   777	                    try:
   778	                        requests, outputs = await state.pipeline.process_streaming_batch()
   779	                        if not outputs:
   780	                            break
   781	                        for output in outputs:
   782	                            if output.stream_id == stream_id:
   783	                                text = output.final_transcript or output.partial_transcript or ""
   784	                                is_final = bool(output.final_transcript)
   785	                                if text.strip():
   786	                                    if is_final:
   787	                                        full_segments.append(text.strip())
   788	                                        last_partial = ""
   789	                                    else:
   790	                                        last_partial = text.strip()
   791	                                    event_data = json.dumps({
   792	                                        "text": text,
   793	                                        "is_final": is_final,
   794	                                        "timestamp": time.time(),
   795	                                    })
   796	                                    yield f"data: {event_data}\n\n"
   797	                    except Exception:
   798	                        break
   799
   800	                # Final event with complete transcription
   801	                combined = " ".join([x for x in [*full_segments, last_partial] if x]).strip()
   802	                yield f"data: {json.dumps({'text': combined, 'is_final': True, 'done': True})}\n\n"
   803
   804	            finally:
   805	                # Cleanup
   806	                if stream_id in state.pipeline._state_pool:
   807	                    del state.pipeline._state_pool[stream_id]
   808	                if stream_id in state.pipeline._streaming_request_generator.streams:
   809	                    state.pipeline._streaming_request_generator.streams.pop(stream_id, None)
   810
   811	        state.metrics["http_requests_ok"] += 1
   812	        return StreamingResponse(
   813	            sse_generator(),
   814	            media_type="text/event-stream",
   815	            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
   816	        )
   817
   818	    except HTTPException:
   819	        raise
   820	    except Exception as e:
   821	        state.metrics["http_requests_err"] += 1
   822	        logger.error(f"HTTP stream error: {e}", exc_info=True)
   823	        raise HTTPException(status_code=500, detail=str(e))
   824
   825
   826	# ============================================================
   827	# WebSocket Streaming
   828	# ============================================================
   829
   830	@app.websocket("/ws")
   831	async def websocket_streaming(ws: WebSocket):
   832	    """
   833	    Production WebSocket endpoint for real-time streaming ASR.
   834
   835	    Protocol:
   836	      Client -> Server: binary audio chunks (raw PCM 16-bit 16kHz mono)
   837	      Client -> Server: text "END" to signal end of audio
   838	      Server -> Client: JSON {"text": "...", "is_final": bool, "timestamp": float}
   839	      Server -> Client: text "END" to confirm stream complete
   840
   841	    Backpressure: Rejects with 1013 when at capacity.
   842	    """
   843	    state.metrics["ws_connections_total"] += 1
   844
   845	    # Backpressure: fast-fail when overloaded
   846	    active = len(state.client_queues)
   847	    if active >= MAX_ACTIVE_STREAMS:
   848	        state.metrics["ws_connections_rejected"] += 1
   849	        await ws.accept()
   850	        await ws.close(code=1013, reason=f"Overloaded: {active}/{MAX_ACTIVE_STREAMS}")
   851	        return
   852
   853	    await ws.accept()
   854	    state.metrics["ws_connections_active"] += 1
   855
   856	    stream_id = None
   857	    try:
   858	        # Start inference loop if needed
   859	        start_inference_loop_if_needed()
   860


Output:
Chunk ID: e21652
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1060
Output:
     1	"""
     2	Vast.ai PyWorker for Hindi ASR serverless endpoint.
     3	Proxies HTTP requests to ASR model server, reports metrics for autoscaling.
     4	"""
     5	import os
     6	import struct
     7	import base64
     8
     9	from vastai import (
    10	    Worker,
    11	    WorkerConfig,
    12	    HandlerConfig,
    13	    BenchmarkConfig,
    14	    LogActionConfig,
    15	)
    16
    17	MODEL_SERVER_URL = "http://127.0.0.1"
    18	MODEL_SERVER_PORT = int(os.environ.get("SERVER_PORT", "8000"))
    19	MODEL_LOG_FILE = os.environ.get("MODEL_LOG", "/var/log/model/server.log")
    20
    21
    22	def asr_workload_calculator(payload):
    23	    """Estimate workload from request. Returns estimated audio seconds."""
    24	    if not isinstance(payload, dict):
    25	        return 10.0
    26	    for key in ("audio_seconds", "audio_duration", "duration"):
    27	        if key in payload:
    28	            try:
    29	                return float(payload[key])
    30	            except Exception:
    31	                pass
    32	    audio_b64 = payload.get("audio_base64") or payload.get("audio_b64") or ""
    33	    if audio_b64:
    34	        try:
    35	            raw_len = len(audio_b64) * 3 / 4
    36	            return max(1.0, raw_len / 32000.0)
    37	        except Exception:
    38	            pass
    39	    return 10.0
    40
    41
    42	def asr_benchmark_generator():
    43	    """Generate benchmark payload: 1s silence WAV (base64 JSON)."""
    44	    sample_rate = 16000
    45	    num_samples = int(sample_rate * 1.0)
    46	    silence = b"\x00\x00" * num_samples
    47	    data_size = len(silence)
    48	    wav_header = struct.pack(
    49	        "<4sI4s4sIHHIIHH4sI",
    50	        b"RIFF", 36 + data_size, b"WAVE", b"fmt ", 16,
    51	        1, 1, sample_rate, sample_rate * 2, 2, 16,
    52	        b"data", data_size,
    53	    )
    54	    wav_bytes = wav_header + silence
    55	    return {
    56	        "audio_base64": base64.b64encode(wav_bytes).decode("ascii"),
    57	        "language": "hi",
    58	    }
    59
    60	def metrics_workload_calculator(_payload):
    61	    """Metrics requests should not affect autoscaling decisions."""
    62	    return 0.1
    63
    64
    65	worker_config = WorkerConfig(
    66	    model_server_url=MODEL_SERVER_URL,
    67	    model_server_port=MODEL_SERVER_PORT,
    68	    model_log_file=MODEL_LOG_FILE,
    69	    model_healthcheck_url=f"{MODEL_SERVER_URL}:{MODEL_SERVER_PORT}/health",
    70	    handlers=[
    71	        HandlerConfig(
    72	            route="/v1/audio/transcriptions",
    73	            allow_parallel_requests=True,
    74	            max_queue_time=120.0,
    75	            workload_calculator=asr_workload_calculator,
    76	            benchmark_config=BenchmarkConfig(
    77	                generator=asr_benchmark_generator,
    78	                runs=4,
    79	                concurrency=1,
    80	            ),
    81	        ),
    82	        HandlerConfig(
    83	            route="/v1/audio/transcriptions/json",
    84	            allow_parallel_requests=True,
    85	            max_queue_time=120.0,
    86	            workload_calculator=asr_workload_calculator,
    87	        ),
    88	        HandlerConfig(
    89	            route="/v1/audio/transcriptions/stream",
    90	            allow_parallel_requests=True,
    91	            max_queue_time=60.0,
    92	            workload_calculator=asr_workload_calculator,
    93	        ),
    94	        HandlerConfig(
    95	            route="/metrics",
    96	            allow_parallel_requests=True,
    97	            max_queue_time=5.0,
    98	            workload_calculator=metrics_workload_calculator,
    99	        ),
   100	    ],
   101	    log_action_config=LogActionConfig(
   102	        on_load=["Application startup complete."],
   103	        on_error=[
   104	            "Traceback (most recent call last):",
   105	            "RuntimeError:",
   106	            "CUDA out of memory",
   107	        ],
   108	        on_info=[
   109	            "Warming up GPU",
   110	            "Pipeline loaded",
   111	        ],
   112	    ),
   113	)
   114
   115	Worker(worker_config).run()


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Inspecting server event loops**


nl -ba production/server.py | sed -n '840,1320p'


Output:
Chunk ID: 2845ff
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 4414
Output:
   840
   841	    Backpressure: Rejects with 1013 when at capacity.
   842	    """
   843	    state.metrics["ws_connections_total"] += 1
   844
   845	    # Backpressure: fast-fail when overloaded
   846	    active = len(state.client_queues)
   847	    if active >= MAX_ACTIVE_STREAMS:
   848	        state.metrics["ws_connections_rejected"] += 1
   849	        await ws.accept()
   850	        await ws.close(code=1013, reason=f"Overloaded: {active}/{MAX_ACTIVE_STREAMS}")
   851	        return
   852
   853	    await ws.accept()
   854	    state.metrics["ws_connections_active"] += 1
   855
   856	    stream_id = None
   857	    try:
   858	        # Start inference loop if needed
   859	        start_inference_loop_if_needed()
   860
   861	        # Open streaming session
   862	        stream_id = state.pipeline.open_streaming_session()
   863	        transcription_queue = await register_client(stream_id)
   864	        elapsed = time.perf_counter() - state.start_time
   865	        logger.info(f"[+{elapsed:.1f}s] WS client {stream_id} connected ({len(state.client_queues)} total)")
   866
   867	    except Exception as e:
   868	        logger.error(f"WS connection setup error: {e}", exc_info=True)
   869	        state.metrics["ws_connections_active"] -= 1
   870	        if stream_id is not None:
   871	            await unregister_client(stream_id)
   872	        if ws.application_state is WebSocketState.CONNECTED:
   873	            await ws.close()
   874	        return
   875
   876	    # Send ready message
   877	    await ws.send_text(json.dumps({"type": "Ready", "id": stream_id}))
   878
   879	    async def recv_loop():
   880	        """Receive audio chunks from client, feed to pipeline"""
   881	        att_ctx = state.cfg.streaming.att_context_size[1] + 1
   882	        frame_threshold = att_ctx * 1280 * 2  # bytes
   883	        audio_buffer = b""
   884
   885	        try:
   886	            while True:
   887	                message = await ws.receive()
   888
   889	                # Text message = control signal
   890	                if "text" in message:
   891	                    text_msg = message["text"]
   892	                    if text_msg == "END":
   893	                        await transcription_queue.put({"type": "marker"})
   894	                        break
   895
   896	                # Binary message = audio data
   897	                if "bytes" in message:
   898	                    raw_bytes = message["bytes"]
   899	                    if isinstance(raw_bytes, str):
   900	                        raw_bytes = raw_bytes.encode("latin-1")
   901	                    audio_buffer += raw_bytes
   902
   903	                    # Feed to pipeline when buffer exceeds frame threshold
   904	                    if len(audio_buffer) >= frame_threshold:
   905	                        # VAD gate: skip silence before GPU
   906	                        if state.vad is not None and not state.vad.is_speech(audio_buffer, stream_id):
   907	                            state.metrics["vad_filtered_chunks"] += 1
   908	                            audio_buffer = b""
   909	                            continue
   910
   911	                        state.pipeline.append_streaming_audio(stream_id, audio_buffer)
   912	                        audio_buffer = b""
   913
   914	                        # Signal inference loop
   915	                        state._batch_ready_event.set()
   916	                        await asyncio.sleep(0)
   917
   918	        except WebSocketDisconnect:
   919	            pass
   920	        except Exception as e:
   921	            logger.error(f"WS recv error stream {stream_id}: {e}")
   922	        finally:
   923	            # Flush remaining buffer
   924	            if audio_buffer:
   925	                try:
   926	                    state.pipeline.append_streaming_audio(stream_id, audio_buffer)
   927	                except Exception:
   928	                    pass
   929
   930	            # Mark stream as ended
   931	            try:
   932	                state.pipeline._streaming_request_generator.streams[stream_id].mark_end()
   933	            except Exception:
   934	                pass
   935
   936	            state._batch_ready_event.set()
   937	            await asyncio.sleep(0.1)
   938	            await transcription_queue.put(None)  # Shutdown signal
   939
   940	    async def send_loop():
   941	        """Send transcription results to client"""
   942	        step_num = 0
   943	        try:
   944	            while True:
   945	                output_dict = await transcription_queue.get()
   946	                if output_dict is None:
   947	                    break
   948
   949	                if output_dict.get("type") == "marker":
   950	                    await ws.send_text("END")
   951	                    break
   952
   953	                json_str = output_dict.get("json_str")
   954	                if json_str:
   955	                    if step_num == 0:
   956	                        elapsed = time.perf_counter() - state.start_time
   957	                        logger.info(f"[+{elapsed:.1f}s] Stream {stream_id}: first token sent")
   958	                    await ws.send_text(json_str)
   959	                    step_num += 1
   960
   961	        except Exception as e:
   962	            logger.error(f"WS send error stream {stream_id}: {e}")
   963
   964	    tasks = []
   965	    try:
   966	        tasks = [
   967	            asyncio.create_task(recv_loop()),
   968	            asyncio.create_task(send_loop()),
   969	        ]
   970	        await asyncio.gather(*tasks, return_exceptions=True)
   971	    except Exception as e:
   972	        logger.error(f"WS error stream {stream_id}: {e}")
   973	    finally:
   974	        state.metrics["ws_connections_active"] -= 1
   975	        await unregister_client(stream_id)
   976	        if ws.application_state is WebSocketState.CONNECTED:
   977	            await ws.close()
   978	        for task in tasks:
   979	            if not task.done():
   980	                task.cancel()
   981	                try:
   982	                    await task
   983	                except asyncio.CancelledError:
   984	                    pass
   985
   986
   987	# ============================================================
   988	# Stream Management + Inference Loop
   989	# ============================================================
   990
   991	async def register_client(stream_id: int) -> asyncio.Queue:
   992	    """Register a streaming client, return its output queue"""
   993	    queue = asyncio.Queue()
   994	    async with state.client_queues_lock:
   995	        state.client_queues[stream_id] = queue
   996	    async with state.stream_timestamps_lock:
   997	        state.stream_audio_timestamps[stream_id] = 0.0
   998	    return queue
   999
  1000
  1001	async def unregister_client(stream_id: int):
  1002	    """Remove client from tracking, mark for cleanup"""
  1003	    try:
  1004	        async with state.client_queues_lock:
  1005	            state.client_queues.pop(stream_id, None)
  1006	        async with state.stream_timestamps_lock:
  1007	            state.stream_audio_timestamps.pop(stream_id, None)
  1008	        if state.vad is not None:
  1009	            state.vad.reset_stream(stream_id)
  1010	        if hasattr(state.pipeline, "_streaming_request_generator"):
  1011	            state.pipeline._streaming_request_generator.close_stream(stream_id)
  1012	        async with state.streams_cleanup_lock:
  1013	            state.streams_pending_cleanup.add(stream_id)
  1014	    except Exception as e:
  1015	        logger.error(f"Unregister error stream {stream_id}: {e}")
  1016
  1017
  1018	async def route_outputs(outputs_with_timestamps):
  1019	    """Route inference outputs to per-client queues"""
  1020	    current_time = time.time()
  1021
  1022	    for output_dict in outputs_with_timestamps:
  1023	        output = output_dict["output"]
  1024	        audio_ts = output_dict["audio_timestamp"]
  1025
  1026	        text = output.current_step_transcript if output.partial_transcript else output.final_transcript
  1027	        is_final = not bool(output.partial_transcript)
  1028
  1029	        result = {
  1030	            "text": text or "",
  1031	            "is_final": is_final,
  1032	            "timestamp": current_time,
  1033	            "audio_timestamp": audio_ts,
  1034	        }
  1035
  1036	        if len(output.final_segments) > 0:
  1037	            result["segment_text"] = output.final_segments[0].text
  1038	            result["segment_start"] = output.final_segments[0].start
  1039	            result["segment_end"] = output.final_segments[0].end
  1040
  1041	        output_dict["json_str"] = json.dumps(result)
  1042
  1043	    # Snapshot queues without holding lock during I/O
  1044	    async with state.client_queues_lock:
  1045	        queues_snapshot = dict(state.client_queues)
  1046
  1047	    for output_dict in outputs_with_timestamps:
  1048	        output = output_dict["output"]
  1049	        queue = queues_snapshot.get(output.stream_id)
  1050	        if queue:
  1051	            try:
  1052	                await queue.put(output_dict)
  1053	            except Exception as e:
  1054	                logger.error(f"Route error stream {output.stream_id}: {e}")
  1055
  1056
  1057	async def cleanup_finished_streams():
  1058	    """Clean up streams fully drained from the request generator"""
  1059	    async with state.streams_cleanup_lock:
  1060	        if not state.streams_pending_cleanup:
  1061	            return
  1062	        active_ids = set(state.pipeline._streaming_request_generator.streams.keys())
  1063	        to_cleanup = state.streams_pending_cleanup - active_ids
  1064	        for sid in to_cleanup:
  1065	            try:
  1066	                if sid in state.pipeline._state_pool:
  1067	                    del state.pipeline._state_pool[sid]
  1068	                if hasattr(state.pipeline, "context_manager") and state.pipeline.context_manager is not None:
  1069	                    try:
  1070	                        state.pipeline.context_manager.reset_slots([sid], [True])
  1071	                    except Exception:
  1072	                        pass
  1073	                state.streams_pending_cleanup.discard(sid)
  1074	            except Exception as e:
  1075	                logger.error(f"Cleanup error stream {sid}: {e}")
  1076
  1077
  1078	async def centralized_inference_loop():
  1079	    """
  1080	    Production inference loop with event-driven batching.
  1081
  1082	    WHY event-driven: fires immediately when data arrives (high load) or
  1083	    after MAX_BATCH_WAIT_MS timeout (low load). 6x better than fixed 300ms sleep.
  1084	    """
  1085	    state.inference_running = True
  1086	    max_wait = MAX_BATCH_WAIT_MS / 1000.0
  1087	    metrics_report_interval = 100
  1088
  1089	    logger.info(f"Inference loop started (max_wait={MAX_BATCH_WAIT_MS}ms)")
  1090
  1091	    while state.inference_running:
  1092	        try:
  1093	            active = len(state.client_queues)
  1094
  1095	            # Idle if no clients
  1096	            if active == 0:
  1097	                try:
  1098	                    await asyncio.wait_for(state._batch_ready_event.wait(), timeout=0.1)
  1099	                    state._batch_ready_event.clear()
  1100	                except asyncio.TimeoutError:
  1101	                    pass
  1102	                continue
  1103
  1104	            # Event-driven wait
  1105	            cycle_start = time.perf_counter()
  1106	            try:
  1107	                await asyncio.wait_for(state._batch_ready_event.wait(), timeout=max_wait)
  1108	            except asyncio.TimeoutError:
  1109	                pass
  1110	            state._batch_ready_event.clear()
  1111
  1112	            # Process ALL ready frames (exhaustive)
  1113	            all_outputs = []
  1114	            async with state.stream_timestamps_lock:
  1115	                timestamps = dict(state.stream_audio_timestamps)
  1116
  1117	            batches_this_cycle = 0
  1118	            while True:
  1119	                try:
  1120	                    batch_start = time.perf_counter()
  1121	                    requests, outputs = await state.pipeline.process_streaming_batch()
  1122	                    gpu_ms = (time.perf_counter() - batch_start) * 1000
  1123
  1124	                    if not outputs:
  1125	                        break
  1126
  1127	                    batches_this_cycle += 1
  1128	                    state.metrics["total_batches"] += 1
  1129	                    state.metrics["total_outputs"] += len(outputs)
  1130	                    state.metrics["total_gpu_ms"] += gpu_ms
  1131	                    state.metrics["batch_sizes"].append(len(outputs))
  1132	                    state.metrics["gpu_times_ms"].append(gpu_ms)
  1133
  1134	                    # Update timestamps
  1135	                    for req in requests:
  1136	                        if hasattr(req, "length"):
  1137	                            sid = req.stream_id
  1138	                            timestamps[sid] = timestamps.get(sid, 0.0) + req.length / 16000.0
  1139
  1140	                    for output in outputs:
  1141	                        all_outputs.append({
  1142	                            "output": output,
  1143	                            "audio_timestamp": timestamps.get(output.stream_id, 0.0),
  1144	                        })
  1145
  1146	                    await asyncio.sleep(0)
  1147
  1148	                except Exception as e:
  1149	                    if type(e).__name__ == "NotEnoughDataException":
  1150	                        if batches_this_cycle == 0:
  1151	                            state.metrics["empty_cycles"] += 1
  1152	                        break
  1153	                    raise
  1154
  1155	            if all_outputs:
  1156	                async with state.stream_timestamps_lock:
  1157	                    for sid, ts in timestamps.items():
  1158	                        if sid in state.stream_audio_timestamps:
  1159	                            state.stream_audio_timestamps[sid] = ts
  1160
  1161	                route_start = time.perf_counter()
  1162	                await route_outputs(all_outputs)
  1163	                state.metrics["total_route_ms"] += (time.perf_counter() - route_start) * 1000
  1164
  1165	                await cleanup_finished_streams()
  1166
  1167	                cycle_ms = (time.perf_counter() - cycle_start) * 1000
  1168	                state.metrics["cycle_times_ms"].append(cycle_ms)
  1169
  1170	                # Periodic metrics log
  1171	                if state.metrics["total_batches"] % metrics_report_interval == 0:
  1172	                    _log_metrics(active)
  1173
  1174	        except Exception as e:
  1175	            logger.error(f"Inference loop error: {e}", exc_info=True)
  1176	            await asyncio.sleep(0.1)
  1177
  1178	    logger.warning("Inference loop stopped")
  1179
  1180
  1181	def _log_metrics(active_clients: int):
  1182	    """Structured metrics log"""
  1183	    m = state.metrics
  1184
  1185	    def pct(arr, p):
  1186	        if not arr:
  1187	            return 0
  1188	        s = sorted(arr)
  1189	        return s[min(int(len(s) * p / 100), len(s) - 1)]
  1190
  1191	    try:
  1192	        vram_used = torch.cuda.memory_allocated() / 1e9
  1193	        vram_total = torch.cuda.get_device_properties(0).total_memory / 1e9
  1194	    except Exception:
  1195	        vram_used = vram_total = 0
  1196
  1197	    elapsed = time.perf_counter() - state.start_time
  1198	    logger.info(
  1199	        f"[+{elapsed:.0f}s] METRICS | streams={active_clients} "
  1200	        f"batches={m['total_batches']} outputs={m['total_outputs']} "
  1201	        f"GPU: p50={pct(m['gpu_times_ms'], 50):.1f}ms p99={pct(m['gpu_times_ms'], 99):.1f}ms | "
  1202	        f"VRAM: {vram_used:.1f}/{vram_total:.0f}GB | "
  1203	        f"vad_skip={m['vad_filtered_chunks']}"
  1204	    )
  1205
  1206
  1207	def start_inference_loop_if_needed():
  1208	    """Start centralized inference loop on first client connection"""
  1209	    if state.inference_task is None:
  1210	        logger.info("Starting inference loop (first client)")
  1211	        state.inference_task = asyncio.create_task(centralized_inference_loop())
  1212	    elif state.inference_task.done():
  1213	        logger.warning("Inference task was done, restarting")
  1214	        state.inference_task = asyncio.create_task(centralized_inference_loop())
  1215
  1216
  1217	# ============================================================
  1218	# Main
  1219	# ============================================================
  1220
  1221	if __name__ == "__main__":
  1222	    logger.info(f"Starting ASR server on port {SERVER_PORT}")
  1223	    logger.info(f"  Model: {MODEL_PATH}")
  1224	    logger.info(f"  Max streams: {MAX_ACTIVE_STREAMS}")
  1225	    logger.info(f"  Batch wait: {MAX_BATCH_WAIT_MS}ms")
  1226	    logger.info(f"  VAD: {'enabled' if ENABLE_VAD else 'disabled'}")
  1227
  1228	    uvicorn.run(
  1229	        app,
  1230	        host="0.0.0.0",
  1231	        port=SERVER_PORT,
  1232	        log_level="info",
  1233	        access_log=False,  # Reduce noise, we have structured logging
  1234	        timeout_keep_alive=300,  # 5min for long WebSocket sessions
  1235	    )


Summary: auto


**Reviewing stress test for relevance**


nl -ba production/test_stress_full.py | sed -n '1,420p'


Output:
Chunk ID: f6372c
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 4089
Output:
     1	#!/usr/bin/env python3
     2	"""
     3	Full-fidelity stress + quality test for Hindi ASR over Vast serverless (HTTP/SSE).
     4
     5	Features:
     6	- Customer-like concurrent clients with multiple requests per client
     7	- Route timing + queue-full detection
     8	- First-token/total latency + server processing time
     9	- Optional FLEURS Hindi audio (real speech) with WER/CER checks
    10	- Time-series snapshots from /metrics and /get_endpoint_workers/
    11	- CSV + JSON outputs suitable for scaling-law fitting
    12	"""
    13
    14	import argparse
    15	import asyncio
    16	import base64
    17	import csv
    18	import io
    19	import json
    20	import math
    21	import os
    22	import random
    23	import re
    24	import struct
    25	import sys
    26	import time
    27	import wave
    28	from collections import Counter
    29	from dataclasses import dataclass, asdict
    30	from datetime import datetime
    31	from pathlib import Path
    32	from typing import Any, Dict, List, Optional, Tuple
    33
    34	import aiohttp
    35	import numpy as np
    36	import requests
    37
    38	try:
    39	    import jiwer
    40	except Exception:
    41	    jiwer = None
    42
    43	VAST_API = "https://console.vast.ai/api/v0"
    44	VAST_RUN = "https://run.vast.ai"
    45
    46
    47	def _env_or_dotenv(name: str) -> Optional[str]:
    48	    if os.environ.get(name):
    49	        return os.environ.get(name)
    50	    env_path = Path(__file__).resolve().parent.parent / ".env"
    51	    if env_path.exists():
    52	        for line in env_path.read_text().splitlines():
    53	            if line.startswith(name + "="):
    54	                return line.split("=", 1)[1].strip()
    55	    return None
    56
    57
    58	def get_endpoint_info(endpoint_name: str, account_key: Optional[str] = None) -> Dict[str, Any]:
    59	    key = account_key or _env_or_dotenv("VAST_KEY")
    60	    if not key:
    61	        raise RuntimeError("VAST_KEY not set; pass --endpoint-key and --endpoint-id")
    62	    r = requests.get(
    63	        f"{VAST_API}/endptjobs/",
    64	        headers={"Accept": "application/json", "Authorization": f"Bearer {key}"},
    65	        timeout=30,
    66	    )
    67	    data = r.json()
    68	    results = data.get("results", []) if isinstance(data, dict) else data
    69	    for ep in results or []:
    70	        if ep.get("endpoint_name") == endpoint_name:
    71	            return ep
    72	    raise RuntimeError(f"endpoint '{endpoint_name}' not found")
    73
    74
    75	async def get_route(session: aiohttp.ClientSession, endpoint_name: str, endpoint_key: str, cost: float) -> Tuple[Optional[Dict[str, Any]], float, Optional[str], bool]:
    76	    t0 = time.perf_counter()
    77	    try:
    78	        async with session.post(
    79	            f"{VAST_RUN}/route/",
    80	            json={"endpoint": endpoint_name, "api_key": endpoint_key, "cost": cost},
    81	            timeout=aiohttp.ClientTimeout(total=15),
    82	        ) as r:
    83	            data = await r.json(content_type=None)
    84	    except Exception as e:
    85	        return None, (time.perf_counter() - t0) * 1000.0, str(e), False
    86
    87	    dt = (time.perf_counter() - t0) * 1000.0
    88	    if isinstance(data, dict) and "url" in data:
    89	        return data, dt, None, False
    90
    91	    status = ""
    92	    if isinstance(data, dict):
    93	        status = str(data.get("status", ""))
    94	    queue_full = "all worker queues full" in status.lower()
    95	    return None, dt, f"route failed: {data}", queue_full
    96
    97
    98	def wrap_auth(route: Dict[str, Any], payload: Dict[str, Any]) -> Dict[str, Any]:
    99	    auth = {k: route[k] for k in ["signature", "cost", "endpoint", "reqnum", "url", "request_idx"] if k in route}
   100	    return {"auth_data": auth, "payload": payload}
   101
   102
   103	def pct(vals: List[float], p: int) -> float:
   104	    if not vals:
   105	        return 0.0
   106	    s = sorted(vals)
   107	    idx = min(int(len(s) * p / 100), len(s) - 1)
   108	    return float(s[idx])
   109
   110
   111	def safe_mean(vals: List[float]) -> float:
   112	    return float(sum(vals) / len(vals)) if vals else 0.0
   113
   114
   115	def normalize_text(text: str) -> str:
   116	    if not text:
   117	        return ""
   118	    text = text.lower().strip()
   119	    text = re.sub(r"<[^>]+>", " ", text)
   120	    text = re.sub(r"[^\u0900-\u097fa-z0-9\s]", " ", text)
   121	    text = re.sub(r"\s+", " ", text).strip()
   122	    return text
   123
   124
   125	def compute_wer_cer(reference: str, hypothesis: str) -> Tuple[Optional[float], Optional[float]]:
   126	    if not jiwer:
   127	        return None, None
   128	    ref = normalize_text(reference)
   129	    hyp = normalize_text(hypothesis)
   130	    if not ref:
   131	        return None, None
   132	    try:
   133	        return float(jiwer.wer(ref, hyp)), float(jiwer.cer(ref, hyp))
   134	    except Exception:
   135	        return None, None
   136
   137
   138	@dataclass
   139	class AudioSample:
   140	    sample_id: str
   141	    duration_s: float
   142	    wav_bytes: bytes
   143	    reference: Optional[str] = None
   144
   145
   146	def _pcm_to_wav_bytes(pcm_int16: np.ndarray, sample_rate: int = 16000) -> bytes:
   147	    bio = io.BytesIO()
   148	    with wave.open(bio, "wb") as wf:
   149	        wf.setnchannels(1)
   150	        wf.setsampwidth(2)
   151	        wf.setframerate(sample_rate)
   152	        wf.writeframes(pcm_int16.astype(np.int16).tobytes())
   153	    return bio.getvalue()
   154
   155
   156	def make_sine_samples(durations: List[float]) -> List[AudioSample]:
   157	    samples: List[AudioSample] = []
   158	    for i, d in enumerate(durations):
   159	        n = int(16000 * d)
   160	        pcm = np.array(
   161	            [
   162	                min(32767, max(-32768, int(16000 * math.sin(2 * math.pi * 440 * k / 16000))))
   163	                for k in range(n)
   164	            ],
   165	            dtype=np.int16,
   166	        )
   167	        wav = _pcm_to_wav_bytes(pcm)
   168	        samples.append(AudioSample(sample_id=f"sine_{i}_{d:.2f}s", duration_s=float(d), wav_bytes=wav, reference=None))
   169	    return samples
   170
   171
   172	def load_fleurs_samples(max_samples: int, min_duration: float, max_duration: float, seed: int) -> List[AudioSample]:
   173	    from datasets import load_dataset
   174
   175	    ds = load_dataset("google/fleurs", "hi_in", split="test")
   176	    rows = list(ds)
   177	    rnd = random.Random(seed)
   178	    rnd.shuffle(rows)
   179
   180	    out: List[AudioSample] = []
   181	    for row in rows:
   182	        audio = row["audio"]
   183	        arr = np.asarray(audio["array"])  # float32 [-1, 1]
   184	        sr = int(audio["sampling_rate"])
   185	        if sr != 16000:
   186	            # FLEURS hi_in is normally 16kHz, but keep explicit guard.
   187	            continue
   188	        duration = len(arr) / float(sr)
   189	        if duration < min_duration or duration > max_duration:
   190	            continue
   191	        pcm = np.clip(arr, -1.0, 1.0)
   192	        pcm = (pcm * 32767.0).astype(np.int16)
   193	        wav = _pcm_to_wav_bytes(pcm, sample_rate=sr)
   194	        out.append(
   195	            AudioSample(
   196	                sample_id=str(row["id"]),
   197	                duration_s=round(duration, 3),
   198	                wav_bytes=wav,
   199	                reference=row.get("transcription"),
   200	            )
   201	        )
   202	        if len(out) >= max_samples:
   203	            break
   204
   205	    if not out:
   206	        raise RuntimeError("No FLEURS samples matched filters")
   207	    return out
   208
   209
   210	@dataclass
   211	class RequestRecord:
   212	    client_idx: int
   213	    req_idx: int
   214	    mode: str
   215	    sample_id: str
   216	    duration_s: float
   217	    route_ms: float
   218	    connect_ms: float
   219	    first_token_ms: float
   220	    total_ms: float
   221	    server_processing_ms: float
   222	    status_code: int
   223	    ok: bool
   224	    queue_full: bool
   225	    error: str
   226	    reference: str
   227	    hypothesis: str
   228	    wer: float
   229	    cer: float
   230
   231
   232	@dataclass
   233	class LevelResult:
   234	    mode: str
   235	    concurrency: int
   236	    requests_per_client: int
   237	    wall_time_s: float
   238	    ok: int
   239	    errors: int
   240	    queue_full_errors: int
   241	    throughput_x: float
   242	    route_ms_p50: float
   243	    route_ms_p95: float
   244	    route_ms_p99: float
   245	    first_token_ms_p50: float
   246	    first_token_ms_p95: float
   247	    first_token_ms_p99: float
   248	    total_ms_p50: float
   249	    total_ms_p95: float
   250	    total_ms_p99: float
   251	    server_proc_ms_p50: float
   252	    server_proc_ms_p95: float
   253	    server_proc_ms_p99: float
   254	    vram_used_gb_max: float
   255	    worker_cur_load_max: float
   256	    worker_cur_load_p95: float
   257	    worker_cur_perf_avg: float
   258	    wer_pct: float
   259	    cer_pct: float
   260	    status_counts: Dict[str, int]
   261
   262
   263	async def fetch_server_metrics_via_endpoint(
   264	    session: aiohttp.ClientSession,
   265	    endpoint_name: str,
   266	    endpoint_key: str,
   267	    route_cost: float,
   268	) -> Dict[str, Any]:
   269	    route, _, err, _ = await get_route(session, endpoint_name, endpoint_key, route_cost)
   270	    if not route:
   271	        return {"error": err}
   272	    worker_url = route["url"].rstrip("/")
   273	    req = wrap_auth(route, {})
   274	    try:
   275	        async with session.post(
   276	            f"{worker_url}/metrics",
   277	            json=req,
   278	            timeout=aiohttp.ClientTimeout(total=12),
   279	        ) as r:
   280	            if r.status == 200:
   281	                return await r.json(content_type=None)
   282	            return {"error": f"metrics status={r.status}"}
   283	    except Exception as e:
   284	        return {"error": str(e)}
   285
   286
   287	async def fetch_endpoint_workers(
   288	    session: aiohttp.ClientSession,
   289	    endpoint_id: int,
   290	    endpoint_key: str,
   291	) -> List[Dict[str, Any]]:
   292	    try:
   293	        async with session.post(
   294	            f"{VAST_RUN}/get_endpoint_workers/",
   295	            json={"id": endpoint_id, "api_key": endpoint_key},
   296	            timeout=aiohttp.ClientTimeout(total=12),
   297	        ) as r:
   298	            data = await r.json(content_type=None)
   299	            if isinstance(data, dict) and "workers" in data:
   300	                return data["workers"]
   301	            return data if isinstance(data, list) else []
   302	    except Exception:
   303	        return []
   304
   305
   306	async def metrics_sampler(
   307	    stop_evt: asyncio.Event,
   308	    interval_s: float,
   309	    session: aiohttp.ClientSession,
   310	    endpoint_name: Optional[str],
   311	    endpoint_key: Optional[str],
   312	    endpoint_id: Optional[int],
   313	    route_cost: float,
   314	    server_samples: List[Dict[str, Any]],
   315	    worker_samples: List[Dict[str, Any]],
   316	):
   317	    while not stop_evt.is_set():
   318	        ts = time.time()
   319	        if endpoint_name and endpoint_key:
   320	            sm = await fetch_server_metrics_via_endpoint(session, endpoint_name, endpoint_key, route_cost)
   321	            server_samples.append({"t": ts, "metrics": sm})
   322	            if endpoint_id is not None:
   323	                workers = await fetch_endpoint_workers(session, endpoint_id, endpoint_key)
   324	                worker_samples.append({"t": ts, "workers": workers})
   325	        await asyncio.sleep(interval_s)
   326
   327
   328	async def run_http_request(
   329	    session: aiohttp.ClientSession,
   330	    sample: AudioSample,
   331	    endpoint_name: Optional[str],
   332	    endpoint_key: Optional[str],
   333	    route_cost: float,
   334	    direct_url: Optional[str],
   335	) -> RequestRecord:
   336	    t0 = time.perf_counter()
   337	    route_ms = 0.0
   338	    queue_full = False
   339	    connect_ms = 0.0
   340	    status_code = 0
   341	    first_token_ms = 0.0
   342	    server_proc_ms = 0.0
   343	    hypothesis = ""
   344	    err = ""
   345
   346	    try:
   347	        if endpoint_name and endpoint_key:
   348	            route, route_ms, rerr, qf = await get_route(session, endpoint_name, endpoint_key, route_cost)
   349	            queue_full = qf
   350	            if not route:
   351	                raise RuntimeError(rerr or "route failed")
   352	            worker_url = route["url"].rstrip("/")
   353	            payload = {
   354	                "audio_base64": base64.b64encode(sample.wav_bytes).decode("ascii"),
   355	                "language": "hi",
   356	                "audio_duration": sample.duration_s,
   357	            }
   358	            req = wrap_auth(route, payload)
   359	            t_conn = time.perf_counter()
   360	            async with session.post(
   361	                f"{worker_url}/v1/audio/transcriptions",
   362	                json=req,
   363	                timeout=aiohttp.ClientTimeout(total=180),
   364	            ) as r:
   365	                connect_ms = (time.perf_counter() - t_conn) * 1000.0
   366	                status_code = int(r.status)
   367	                if r.status in (404, 422):
   368	                    route2, route_ms2, rerr2, qf2 = await get_route(session, endpoint_name, endpoint_key, route_cost)
   369	                    route_ms += route_ms2
   370	                    queue_full = queue_full or qf2
   371	                    if not route2:
   372	                        raise RuntimeError(rerr2 or "route fallback failed")
   373	                    req2 = wrap_auth(route2, payload)
   374	                    worker_url2 = route2["url"].rstrip("/")
   375	                    async with session.post(
   376	                        f"{worker_url2}/v1/audio/transcriptions/json",
   377	                        json=req2,
   378	                        timeout=aiohttp.ClientTimeout(total=180),
   379	                    ) as r2:
   380	                        status_code = int(r2.status)
   381	                        data = await r2.json(content_type=None)
   382	                else:
   383	                    data = await r.json(content_type=None)
   384	        else:
   385	            if not direct_url:
   386	                raise RuntimeError("direct_url required when endpoint is not set")
   387	            form = aiohttp.FormData()
   388	            form.add_field("file", sample.wav_bytes, filename="sample.wav", content_type="audio/wav")
   389	            form.add_field("language", "hi")
   390	            t_conn = time.perf_counter()
   391	            async with session.post(
   392	                f"{direct_url.rstrip('/')}/v1/audio/transcriptions",
   393	                data=form,
   394	                timeout=aiohttp.ClientTimeout(total=180),
   395	            ) as r:
   396	                connect_ms = (time.perf_counter() - t_conn) * 1000.0
   397	                status_code = int(r.status)
   398	                data = await r.json(content_type=None)
   399
   400	        if status_code != 200:
   401	            err = f"HTTP {status_code}: {str(data)[:400]}"
   402	        else:
   403	            hypothesis = str(data.get("text", "") or "")
   404	            if "processing_time" in data:
   405	                try:
   406	                    server_proc_ms = float(data["processing_time"]) * 1000.0
   407	                except Exception:
   408	                    server_proc_ms = 0.0
   409
   410	    except Exception as e:
   411	        err = str(e)
   412
   413	    t1 = time.perf_counter()
   414	    total_ms = (t1 - t0) * 1000.0
   415	    first_token_ms = total_ms  # HTTP batch has no token stream
   416
   417	    ref = sample.reference or ""
   418	    wer_val, cer_val = compute_wer_cer(ref, hypothesis)
   419
   420	    return RequestRecord(


I found `production/test_stress_full.py`, which is a much better baseline for “full metrics + quality” than `test_stress.py`; it already includes FLEURS WER/CER, route/queue-full stats, and time-series metric sampling. I’m reading the rest now and then I’ll patch it for websocket mode with dynamic port discovery so we can run apples-to-apples benchmarks.


nl -ba production/test_stress_full.py | sed -n '420,980p'


Output:
Chunk ID: 444c4e
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 6025
Output:
   420	    return RequestRecord(
   421	        client_idx=-1,
   422	        req_idx=-1,
   423	        mode="http",
   424	        sample_id=sample.sample_id,
   425	        duration_s=sample.duration_s,
   426	        route_ms=route_ms,
   427	        connect_ms=connect_ms,
   428	        first_token_ms=first_token_ms,
   429	        total_ms=total_ms,
   430	        server_processing_ms=server_proc_ms,
   431	        status_code=status_code,
   432	        ok=(err == "" and status_code == 200),
   433	        queue_full=queue_full,
   434	        error=err,
   435	        reference=ref,
   436	        hypothesis=hypothesis,
   437	        wer=float("nan") if wer_val is None else wer_val,
   438	        cer=float("nan") if cer_val is None else cer_val,
   439	    )
   440
   441
   442	async def run_sse_request(
   443	    session: aiohttp.ClientSession,
   444	    sample: AudioSample,
   445	    endpoint_name: Optional[str],
   446	    endpoint_key: Optional[str],
   447	    route_cost: float,
   448	    direct_url: Optional[str],
   449	) -> RequestRecord:
   450	    t0 = time.perf_counter()
   451	    route_ms = 0.0
   452	    queue_full = False
   453	    connect_ms = 0.0
   454	    status_code = 0
   455	    first_token_ms = 0.0
   456	    hypothesis = ""
   457	    err = ""
   458
   459	    try:
   460	        if endpoint_name and endpoint_key:
   461	            route, route_ms, rerr, qf = await get_route(session, endpoint_name, endpoint_key, route_cost)
   462	            queue_full = qf
   463	            if not route:
   464	                raise RuntimeError(rerr or "route failed")
   465	            worker_url = route["url"].rstrip("/")
   466	            payload = {
   467	                "audio_base64": base64.b64encode(sample.wav_bytes).decode("ascii"),
   468	                "language": "hi",
   469	                "chunk_duration_ms": 160,
   470	                "audio_duration": sample.duration_s,
   471	            }
   472	            req = wrap_auth(route, payload)
   473	            t_conn = time.perf_counter()
   474	            async with session.post(
   475	                f"{worker_url}/v1/audio/transcriptions/stream",
   476	                json=req,
   477	                timeout=aiohttp.ClientTimeout(total=180),
   478	            ) as r:
   479	                connect_ms = (time.perf_counter() - t_conn) * 1000.0
   480	                status_code = int(r.status)
   481	                if status_code != 200:
   482	                    body = await r.text()
   483	                    raise RuntimeError(f"SSE {status_code}: {body[:200]}")
   484
   485	                got_first = False
   486	                saw_any_event = False
   487	                saw_done = False
   488	                last_text = ""
   489	                async for line in r.content:
   490	                    l = line.decode(errors="ignore").strip()
   491	                    if not l.startswith("data: "):
   492	                        continue
   493	                    try:
   494	                        d = json.loads(l[6:])
   495	                    except Exception:
   496	                        continue
   497	                    saw_any_event = True
   498	                    txt = str(d.get("text", "") or "")
   499	                    if txt:
   500	                        last_text = txt
   501	                        if not got_first:
   502	                            first_token_ms = (time.perf_counter() - t0) * 1000.0
   503	                            got_first = True
   504	                    if d.get("done"):
   505	                        saw_done = True
   506	                        break
   507	                hypothesis = last_text
   508	                if not saw_any_event or not saw_done:
   509	                    raise RuntimeError("sse stream incomplete: no terminal done event")
   510	        else:
   511	            if not direct_url:
   512	                raise RuntimeError("direct_url required when endpoint is not set")
   513	            form = aiohttp.FormData()
   514	            form.add_field("file", sample.wav_bytes, filename="sample.wav", content_type="audio/wav")
   515	            t_conn = time.perf_counter()
   516	            async with session.post(
   517	                f"{direct_url.rstrip('/')}/v1/audio/transcriptions/stream",
   518	                data=form,
   519	                timeout=aiohttp.ClientTimeout(total=180),
   520	            ) as r:
   521	                connect_ms = (time.perf_counter() - t_conn) * 1000.0
   522	                status_code = int(r.status)
   523	                if status_code != 200:
   524	                    body = await r.text()
   525	                    raise RuntimeError(f"SSE {status_code}: {body[:200]}")
   526	                got_first = False
   527	                saw_any_event = False
   528	                saw_done = False
   529	                last_text = ""
   530	                async for line in r.content:
   531	                    l = line.decode(errors="ignore").strip()
   532	                    if not l.startswith("data: "):
   533	                        continue
   534	                    try:
   535	                        d = json.loads(l[6:])
   536	                    except Exception:
   537	                        continue
   538	                    saw_any_event = True
   539	                    txt = str(d.get("text", "") or "")
   540	                    if txt:
   541	                        last_text = txt
   542	                        if not got_first:
   543	                            first_token_ms = (time.perf_counter() - t0) * 1000.0
   544	                            got_first = True
   545	                    if d.get("done"):
   546	                        saw_done = True
   547	                        break
   548	                hypothesis = last_text
   549	                if not saw_any_event or not saw_done:
   550	                    raise RuntimeError("sse stream incomplete: no terminal done event")
   551	    except Exception as e:
   552	        err = str(e)
   553
   554	    t1 = time.perf_counter()
   555	    total_ms = (t1 - t0) * 1000.0
   556	    if first_token_ms <= 0:
   557	        first_token_ms = total_ms
   558
   559	    ref = sample.reference or ""
   560	    wer_val, cer_val = compute_wer_cer(ref, hypothesis)
   561
   562	    return RequestRecord(
   563	        client_idx=-1,
   564	        req_idx=-1,
   565	        mode="sse",
   566	        sample_id=sample.sample_id,
   567	        duration_s=sample.duration_s,
   568	        route_ms=route_ms,
   569	        connect_ms=connect_ms,
   570	        first_token_ms=first_token_ms,
   571	        total_ms=total_ms,
   572	        server_processing_ms=0.0,
   573	        status_code=status_code,
   574	        ok=(err == "" and status_code == 200),
   575	        queue_full=queue_full,
   576	        error=err,
   577	        reference=ref,
   578	        hypothesis=hypothesis,
   579	        wer=float("nan") if wer_val is None else wer_val,
   580	        cer=float("nan") if cer_val is None else cer_val,
   581	    )
   582
   583
   584	def _series_vram_max(server_samples: List[Dict[str, Any]]) -> float:
   585	    vals: List[float] = []
   586	    for s in server_samples:
   587	        m = s.get("metrics", {})
   588	        if isinstance(m, dict) and "vram_used_gb" in m:
   589	            try:
   590	                vals.append(float(m.get("vram_used_gb", 0.0)))
   591	            except Exception:
   592	                pass
   593	    return max(vals) if vals else 0.0
   594
   595
   596	def _worker_stats(worker_samples: List[Dict[str, Any]]) -> Tuple[float, float, float]:
   597	    loads: List[float] = []
   598	    perfs: List[float] = []
   599	    for s in worker_samples:
   600	        workers = s.get("workers") or []
   601	        for w in workers:
   602	            try:
   603	                loads.append(float(w.get("cur_load", 0.0)))
   604	                perfs.append(float(w.get("cur_perf", 0.0)))
   605	            except Exception:
   606	                pass
   607	    return (max(loads) if loads else 0.0, pct(loads, 95), safe_mean(perfs))
   608
   609
   610	def summarize_level(
   611	    mode: str,
   612	    concurrency: int,
   613	    requests_per_client: int,
   614	    wall_time_s: float,
   615	    records: List[RequestRecord],
   616	    server_samples: List[Dict[str, Any]],
   617	    worker_samples: List[Dict[str, Any]],
   618	) -> LevelResult:
   619	    ok_recs = [r for r in records if r.ok]
   620	    route_ms = [r.route_ms for r in records if r.route_ms > 0]
   621	    first_ms = [r.first_token_ms for r in ok_recs]
   622	    total_ms = [r.total_ms for r in ok_recs]
   623	    proc_ms = [r.server_processing_ms for r in ok_recs if r.server_processing_ms > 0]
   624
   625	    total_audio_ok = sum(r.duration_s for r in ok_recs)
   626	    throughput_x = (total_audio_ok / wall_time_s) if wall_time_s > 0 else 0.0
   627
   628	    wer_vals = [r.wer for r in ok_recs if r.reference and not math.isnan(r.wer)]
   629	    cer_vals = [r.cer for r in ok_recs if r.reference and not math.isnan(r.cer)]
   630
   631	    status_counts: Counter[str] = Counter()
   632	    for r in records:
   633	        k = str(r.status_code if r.status_code else "exception")
   634	        status_counts[k] += 1
   635
   636	    qf = sum(1 for r in records if r.queue_full)
   637
   638	    vram_max = _series_vram_max(server_samples)
   639	    worker_load_max, worker_load_p95, worker_perf_avg = _worker_stats(worker_samples)
   640
   641	    return LevelResult(
   642	        mode=mode,
   643	        concurrency=concurrency,
   644	        requests_per_client=requests_per_client,
   645	        wall_time_s=round(wall_time_s, 3),
   646	        ok=len(ok_recs),
   647	        errors=len(records) - len(ok_recs),
   648	        queue_full_errors=qf,
   649	        throughput_x=round(throughput_x, 3),
   650	        route_ms_p50=round(pct(route_ms, 50), 2),
   651	        route_ms_p95=round(pct(route_ms, 95), 2),
   652	        route_ms_p99=round(pct(route_ms, 99), 2),
   653	        first_token_ms_p50=round(pct(first_ms, 50), 2),
   654	        first_token_ms_p95=round(pct(first_ms, 95), 2),
   655	        first_token_ms_p99=round(pct(first_ms, 99), 2),
   656	        total_ms_p50=round(pct(total_ms, 50), 2),
   657	        total_ms_p95=round(pct(total_ms, 95), 2),
   658	        total_ms_p99=round(pct(total_ms, 99), 2),
   659	        server_proc_ms_p50=round(pct(proc_ms, 50), 2),
   660	        server_proc_ms_p95=round(pct(proc_ms, 95), 2),
   661	        server_proc_ms_p99=round(pct(proc_ms, 99), 2),
   662	        vram_used_gb_max=round(vram_max, 2),
   663	        worker_cur_load_max=round(worker_load_max, 3),
   664	        worker_cur_load_p95=round(worker_load_p95, 3),
   665	        worker_cur_perf_avg=round(worker_perf_avg, 4),
   666	        wer_pct=round(100.0 * safe_mean(wer_vals), 3) if wer_vals else 0.0,
   667	        cer_pct=round(100.0 * safe_mean(cer_vals), 3) if cer_vals else 0.0,
   668	        status_counts=dict(status_counts),
   669	    )
   670
   671
   672	async def run_level(
   673	    mode: str,
   674	    concurrency: int,
   675	    requests_per_client: int,
   676	    samples: List[AudioSample],
   677	    endpoint_name: Optional[str],
   678	    endpoint_key: Optional[str],
   679	    endpoint_id: Optional[int],
   680	    route_cost: float,
   681	    metrics_interval: float,
   682	    direct_url: Optional[str],
   683	    shuffle_samples: bool,
   684	    seed: int,
   685	    inter_request_delay: float,
   686	):
   687	    print("\n" + "=" * 90)
   688	    print(f"Level mode={mode} c={concurrency} reqs/client={requests_per_client}")
   689	    print("=" * 90)
   690
   691	    pool = list(samples)
   692	    if shuffle_samples:
   693	        rnd = random.Random(seed + concurrency)
   694	        rnd.shuffle(pool)
   695
   696	    sample_idx = 0
   697	    sample_lock = asyncio.Lock()
   698
   699	    async def pick_sample() -> AudioSample:
   700	        nonlocal sample_idx
   701	        async with sample_lock:
   702	            s = pool[sample_idx % len(pool)]
   703	            sample_idx += 1
   704	            return s
   705
   706	    timeout = aiohttp.ClientTimeout(total=220)
   707	    connector = aiohttp.TCPConnector(limit=max(200, concurrency * 8), ttl_dns_cache=300)
   708
   709	    records: List[RequestRecord] = []
   710	    server_samples: List[Dict[str, Any]] = []
   711	    worker_samples: List[Dict[str, Any]] = []
   712
   713	    stop_evt = asyncio.Event()
   714
   715	    async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
   716	        sampler_task = asyncio.create_task(
   717	            metrics_sampler(
   718	                stop_evt,
   719	                metrics_interval,
   720	                session,
   721	                endpoint_name,
   722	                endpoint_key,
   723	                endpoint_id,
   724	                route_cost,
   725	                server_samples,
   726	                worker_samples,
   727	            )
   728	        )
   729
   730	        t0 = time.perf_counter()
   731
   732	        async def client_worker(client_idx: int):
   733	            for req_idx in range(requests_per_client):
   734	                sample = await pick_sample()
   735	                if mode == "http":
   736	                    rec = await run_http_request(
   737	                        session,
   738	                        sample,
   739	                        endpoint_name,
   740	                        endpoint_key,
   741	                        route_cost,
   742	                        direct_url,
   743	                    )
   744	                else:
   745	                    rec = await run_sse_request(
   746	                        session,
   747	                        sample,
   748	                        endpoint_name,
   749	                        endpoint_key,
   750	                        route_cost,
   751	                        direct_url,
   752	                    )
   753	                rec.client_idx = client_idx
   754	                rec.req_idx = req_idx
   755	                records.append(rec)
   756
   757	                if inter_request_delay > 0 and req_idx + 1 < requests_per_client:
   758	                    await asyncio.sleep(inter_request_delay)
   759
   760	        await asyncio.gather(*(client_worker(i) for i in range(concurrency)))
   761	        t1 = time.perf_counter()
   762
   763	        stop_evt.set()
   764	        try:
   765	            await asyncio.wait_for(sampler_task, timeout=10)
   766	        except Exception:
   767	            sampler_task.cancel()
   768
   769	    wall = t1 - t0
   770	    summary = summarize_level(
   771	        mode,
   772	        concurrency,
   773	        requests_per_client,
   774	        wall,
   775	        records,
   776	        server_samples,
   777	        worker_samples,
   778	    )
   779
   780	    print(
   781	        f"OK={summary.ok}/{len(records)} err={summary.errors} qfull={summary.queue_full_errors} "
   782	        f"tput={summary.throughput_x:.2f}x total_p95={summary.total_ms_p95:.0f}ms "
   783	        f"ft_p95={summary.first_token_ms_p95:.0f}ms wer={summary.wer_pct:.2f}%"
   784	    )
   785
   786	    return {
   787	        "summary": asdict(summary),
   788	        "records": [asdict(r) for r in records],
   789	        "server_samples": server_samples,
   790	        "worker_samples": worker_samples,
   791	    }
   792
   793
   794	def write_outputs(results: List[Dict[str, Any]], out_prefix: str):
   795	    out_dir = Path(__file__).resolve().parent.parent / "benchmark_results"
   796	    out_dir.mkdir(parents=True, exist_ok=True)
   797
   798	    csv_path = out_dir / f"{out_prefix}.csv"
   799	    json_path = out_dir / f"{out_prefix}.json"
   800	    details_csv_path = out_dir / f"{out_prefix}_details.csv"
   801
   802	    summary_fields = [
   803	        "mode",
   804	        "concurrency",
   805	        "requests_per_client",
   806	        "wall_time_s",
   807	        "ok",
   808	        "errors",
   809	        "queue_full_errors",
   810	        "throughput_x",
   811	        "route_ms_p50",
   812	        "route_ms_p95",
   813	        "route_ms_p99",
   814	        "first_token_ms_p50",
   815	        "first_token_ms_p95",
   816	        "first_token_ms_p99",
   817	        "total_ms_p50",
   818	        "total_ms_p95",
   819	        "total_ms_p99",
   820	        "server_proc_ms_p50",
   821	        "server_proc_ms_p95",
   822	        "server_proc_ms_p99",
   823	        "vram_used_gb_max",
   824	        "worker_cur_load_max",
   825	        "worker_cur_load_p95",
   826	        "worker_cur_perf_avg",
   827	        "wer_pct",
   828	        "cer_pct",
   829	        "status_counts",
   830	    ]
   831
   832	    with csv_path.open("w", newline="") as f:
   833	        w = csv.DictWriter(f, fieldnames=summary_fields)
   834	        w.writeheader()
   835	        for item in results:
   836	            row = dict(item["summary"])
   837	            row["status_counts"] = json.dumps(row.get("status_counts", {}), ensure_ascii=False)
   838	            w.writerow(row)
   839
   840	    with details_csv_path.open("w", newline="") as f:
   841	        fields = [
   842	            "mode",
   843	            "concurrency",
   844	            "client_idx",
   845	            "req_idx",
   846	            "sample_id",
   847	            "duration_s",
   848	            "route_ms",
   849	            "connect_ms",
   850	            "first_token_ms",
   851	            "total_ms",
   852	            "server_processing_ms",
   853	            "status_code",
   854	            "ok",
   855	            "queue_full",
   856	            "error",
   857	            "wer",
   858	            "cer",
   859	            "reference",
   860	            "hypothesis",
   861	        ]
   862	        w = csv.DictWriter(f, fieldnames=fields)
   863	        w.writeheader()
   864	        for item in results:
   865	            s = item["summary"]
   866	            for rec in item["records"]:
   867	                w.writerow(
   868	                    {
   869	                        "mode": s["mode"],
   870	                        "concurrency": s["concurrency"],
   871	                        **{k: rec.get(k, "") for k in fields if k not in ("mode", "concurrency")},
   872	                    }
   873	                )
   874
   875	    with json_path.open("w") as f:
   876	        json.dump(results, f, ensure_ascii=False, indent=2)
   877
   878	    return str(csv_path), str(details_csv_path), str(json_path)
   879
   880
   881	async def main_async():
   882	    p = argparse.ArgumentParser(description="Full stress + quality tester for Hindi ASR")
   883	    p.add_argument("--url", default=None, help="Direct URL, e.g. http://ip:8000")
   884	    p.add_argument("--endpoint", default=None, help="Vast endpoint name")
   885	    p.add_argument("--endpoint-key", default=None, help="Endpoint API key")
   886	    p.add_argument("--endpoint-id", type=int, default=None, help="Endpoint id")
   887	    p.add_argument("--route-cost", type=float, default=10.0)
   888	    p.add_argument("--mode", choices=["http", "sse", "both"], default="both")
   889	    p.add_argument("--levels", default="1,5,10,25,50,75,100")
   890	    p.add_argument("--requests-per-client", type=int, default=1)
   891	    p.add_argument("--cooldown", type=float, default=5.0)
   892	    p.add_argument("--metrics-interval", type=float, default=2.0)
   893	    p.add_argument("--inter-request-delay", type=float, default=0.0)
   894
   895	    p.add_argument("--audio-source", choices=["sine", "fleurs"], default="fleurs")
   896	    p.add_argument("--audio-duration", type=float, default=10.0, help="Used for sine source")
   897	    p.add_argument("--audio-durations", default="", help="Comma-separated durations for sine source")
   898	    p.add_argument("--sample-pool", type=int, default=120, help="Pool size for fleurs samples")
   899	    p.add_argument("--min-duration", type=float, default=3.0)
   900	    p.add_argument("--max-duration", type=float, default=20.0)
   901	    p.add_argument("--shuffle-samples", action="store_true")
   902	    p.add_argument("--seed", type=int, default=42)
   903	    p.add_argument("--output-prefix", default="")
   904	    args = p.parse_args()
   905
   906	    if not args.url and not args.endpoint:
   907	        print("ERROR: pass --url or --endpoint")
   908	        sys.exit(2)
   909
   910	    endpoint_name = args.endpoint
   911	    endpoint_key = args.endpoint_key
   912	    endpoint_id = args.endpoint_id
   913	    direct_url = args.url.rstrip("/") if args.url else None
   914
   915	    if endpoint_name:
   916	        if not endpoint_key or not endpoint_id:
   917	            info = get_endpoint_info(endpoint_name)
   918	            endpoint_key = endpoint_key or info.get("api_key")
   919	            endpoint_id = endpoint_id or info.get("id")
   920	        if not endpoint_key or endpoint_id is None:
   921	            raise RuntimeError("Could not resolve endpoint_key/endpoint_id")
   922
   923	    if args.audio_source == "sine":
   924	        if args.audio_durations:
   925	            durations = [float(x) for x in args.audio_durations.split(",") if x.strip()]
   926	        else:
   927	            durations = [args.audio_duration]
   928	        samples = make_sine_samples(durations)
   929	    else:
   930	        samples = load_fleurs_samples(
   931	            max_samples=args.sample_pool,
   932	            min_duration=args.min_duration,
   933	            max_duration=args.max_duration,
   934	            seed=args.seed,
   935	        )
   936
   937	    levels = [int(x) for x in args.levels.split(",") if x.strip()]
   938	    modes = [args.mode] if args.mode != "both" else ["http", "sse"]
   939
   940	    print("=== Full Stress Test ===")
   941	    print(f"Target: {'endpoint ' + endpoint_name if endpoint_name else direct_url}")
   942	    print(f"Modes: {modes}")
   943	    print(f"Levels: {levels}")
   944	    print(f"Audio source: {args.audio_source} (pool={len(samples)})")
   945	    print(f"Requests/client: {args.requests_per_client}")
   946
   947	    results: List[Dict[str, Any]] = []
   948	    for mode in modes:
   949	        for lvl in levels:
   950	            item = await run_level(
   951	                mode=mode,
   952	                concurrency=lvl,
   953	                requests_per_client=args.requests_per_client,
   954	                samples=samples,
   955	                endpoint_name=endpoint_name,
   956	                endpoint_key=endpoint_key,
   957	                endpoint_id=endpoint_id,
   958	                route_cost=args.route_cost,
   959	                metrics_interval=args.metrics_interval,
   960	                direct_url=direct_url,
   961	                shuffle_samples=args.shuffle_samples,
   962	                seed=args.seed,
   963	                inter_request_delay=args.inter_request_delay,
   964	            )
   965	            results.append(item)
   966	            if lvl != levels[-1]:
   967	                await asyncio.sleep(args.cooldown)
   968
   969	    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
   970	    prefix = args.output_prefix or f"stress_full_{ts}"
   971	    csv_path, details_path, json_path = write_outputs(results, prefix)
   972
   973	    print("\n" + "=" * 90)
   974	    print("SUMMARY")
   975	    print("=" * 90)
   976	    for item in results:
   977	        s = item["summary"]
   978	        print(
   979	            f"{s['mode']:>4} c={s['concurrency']:>3} ok={s['ok']:>4}/{s['concurrency'] * s['requests_per_client']:<4} "
   980	            f"err={s['errors']:>3} qfull={s['queue_full_errors']:>3} "


nl -ba production/test_stress_full.py | sed -n '980,1120p'


Output:
Chunk ID: 9cc994
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 143
Output:
   980	            f"err={s['errors']:>3} qfull={s['queue_full_errors']:>3} "
   981	            f"p95_total={s['total_ms_p95']:>8.1f}ms p95_ft={s['first_token_ms_p95']:>8.1f}ms "
   982	            f"tput={s['throughput_x']:>6.2f}x wer={s['wer_pct']:>6.2f}%"
   983	        )
   984
   985	    print(f"\nSummary CSV : {csv_path}")
   986	    print(f"Details CSV : {details_path}")
   987	    print(f"Full JSON   : {json_path}")
   988
   989
   990	def main():
   991	    asyncio.run(main_async())
   992
   993
   994	if __name__ == "__main__":
   995	    main()


**Assessing deployment and endpoints**


`test_stress_full.py` confirms your quality/metrics path is strong for HTTP/SSE, but it currently has no websocket mode, so we’re not yet measuring the same conditions as NVIDIA’s WS scaling post. I’m now checking live Vast endpoint/template state to ensure WS port exposure and then I’ll implement WS benchmarking mode in the full stress harness.


Output:
Chunk ID: e1c653
Wall time: 1.2405 seconds
Process exited with code 0
Original token count: 90
Output:
Endpoint: hindi-asr-prod (id=11967)
  Workers: 7
    id=31138896 status=stopped load=0.0
    id=31140799 status=idle load=10.0
    id=31139135 status=stopped load=0.0
    id=31140874 status=idle load=0.0
    id=31140781 status=stopped load=0.0
    id=31138893 status=stopped load=0.0
    id=31141312 status=stopped load=0.0
  Route: http://79.112.17.186:58896


Output:
Chunk ID: a88170
Wall time: 0.6488 seconds
Process exited with code 0
Original token count: 777
Output:
[
  {
    "cur_load": 0.0,
    "cur_load_rolling_avg": 1.8714969669012166e-39,
    "cur_perf": 0.0,
    "disk_usage": 4.6541290283203125,
    "dlperf": 283.67722901688677,
    "id": 31138896,
    "loaded_at": 0.0,
    "measured_perf": 60.0201953645223,
    "new_load": 0.0,
    "outdated": false,
    "perf": 15.646883964538574,
    "reliability": 0.7145499817904999,
    "reqs_working": 0,
    "started_at": 0.0,
    "status": "stopped"
  },
  {
    "cur_load": 10.0,
    "cur_load_rolling_avg": 0.5000000000524308,
    "cur_perf": 1.384845306740324,
    "disk_usage": 4.634521484375,
    "dlperf": 377.7898826488518,
    "id": 31140799,
    "loaded_at": 1770680585.2953186,
    "measured_perf": 63.789288988108794,
    "new_load": 0.0,
    "outdated": false,
    "perf": 48.291831970214844,
    "reliability": 0.932784850359511,
    "reqs_working": 1,
    "started_at": 1770680401.1913676,
    "status": "idle"
  },
  {
    "cur_load": 0.0,
    "cur_load_rolling_avg": 3.9277717143200046e-44,
    "cur_perf": 0.0,
    "disk_usage": 4.634525299072266,
    "dlperf": 283.48244628500436,
    "id": 31139135,
    "loaded_at": 0.0,
    "measured_perf": 62.81435810315692,
    "new_load": 0.0,
    "outdated": false,
    "perf": 9.196084022521973,
    "reliability": 0.6185659363701174,
    "reqs_working": 0,
    "started_at": 0.0,
    "status": "stopped"
  },
  {
    "cur_load": 0.0,
    "cur_load_rolling_avg": 2.6887600239850274e-11,
    "cur_perf": 0.3540175691424535,
    "disk_usage": 4.634468078613281,
    "dlperf": 282.7617138399875,
    "id": 31140874,
    "loaded_at": 1770680754.6968148,
    "measured_perf": 31.88181371871845,
    "new_load": 0.0,
    "outdated": false,
    "perf": 31.881813049316406,
    "reliability": 0.9999999999999994,
    "reqs_working": 0,
    "started_at": 1770680688.1913676,
    "status": "idle"
  },
  {
    "cur_load": 0.0,
    "cur_load_rolling_avg": 0.0,
    "cur_perf": 0.0,
    "disk_usage": 4.644680023193359,
    "dlperf": 279.33293848751566,
    "id": 31140781,
    "loaded_at": 0.0,
    "measured_perf": 27.557987324331773,
    "new_load": 0.0,
    "outdated": false,
    "perf": 27.557987213134766,
    "reliability": 1.0,
    "reqs_working": 0,
    "started_at": 0.0,
    "status": "stopped"
  },
  {
    "cur_load": 0.0,
    "cur_load_rolling_avg": 2.7429094786517444e-44,
    "cur_perf": 2.054178697149056,
    "disk_usage": 4.808258056640625,
    "dlperf": 283.1902643586163,
    "id": 31138893,
    "loaded_at": 0.0,
    "measured_perf": 64.446012519946,
    "new_load": 0.0,
    "outdated": false,
    "perf": 11.66126823425293,
    "reliability": 0.6522099642728176,
    "reqs_working": 0,
    "started_at": 0.0,
    "status": "stopped"
  },
  {
    "cur_load": 0.0,
    "cur_load_rolling_avg": 0.0,
    "cur_perf": 0.0,
    "disk_usage": 4.654293060302734,
    "dlperf": 362.4305778566506,
    "id": 31141312,
    "loaded_at": 0.0,
    "measured_perf": 29.300630694858146,
    "new_load": 0.0,
    "outdated": false,
    "perf": 29.300630569458008,
    "reliability": 1.0,
    "reqs_working": 0,
    "started_at": 0.0,
    "status": "stopped"
  }
]


Output:
Chunk ID: 719031
Wall time: 0.4394 seconds
Process exited with code 0
Original token count: 138
Output:
{
  "__request_id": "ae9ddc8f-c69f-44b3-9d50-e0c0b26e9c50",
  "cost": 10.0,
  "endpoint": "hindi-asr-prod",
  "reqnum": 2,
  "request_idx": 4159,
  "signature": "B2L/HApuDz/im5Vwp1ZORKrO6f7CnyZvCULUyvLGqs2G4Z8ZAwYQKTZjljabK7XaADeHJdnygNh1YlMjQysNJu36A3dIcKcoCqsG0uCBsERlqwvyo51YfyRJZHtNE5yn/jFgxnITOh3QHuOE9f4BormOfFJtkU44FlLT/4QT9HtgUqQcNxacsgk0CahZWGv2yNQmdcg+VsZ+b6RRPJypWpOCpLTeEKY9pbaXHhWV7hOIWUF0pjaZP3/68k8UxjNMRtulilbwIC4CUsB2GQ7yV3eVpexHKryMVRpfC5tdZ0OXHuVklSzT327wMS466JE9L+LeYN38R+jw7tU6sUGlPQ==",
  "url": "http://104.189.178.116:2273"
}


**Planning environment inspection and API querying**


python3 - <<'PY'
import os,requests,json
from pathlib import Path
# load .env
for line in Path('.env').read_text().splitlines():
    if '=' in line and not line.strip().startswith('#'):
        k,v=line.split('=',1);os.environ.setdefault(k.strip(),v.strip())
key=os.environ.get('VAST_KEY')
api='https://console.vast.ai/api/v0/endptjobs/'
r=requests.get(api,headers={'Accept':'application/json','Authorization':f'Bearer {key}'},timeout=30)
print('status',r.status_code)
d=r.json();results=d.get('results',[]) if isinstance(d,dict) else d
for ep in results:
    print(ep.get('id'),ep.get('endpoint_name'),ep.get('status'),'workers',ep.get('active_workers'), 'template', ep.get('template_hash'))
PY


Output:
Chunk ID: cd7b4a
Wall time: 0.2575 seconds
Process exited with code 0
Original token count: 31
Output:
status 200
11967 hindi-asr-prod None workers None template None
11995 hindi-asr-ws-bench None workers None template None


**Planning workergroups template inspection**


I verified both endpoints exist: `hindi-asr-prod` and `hindi-asr-ws-bench`. I’m now pulling their workergroup/template configs to confirm which one has dual-port mapping (`8000` + `18000`) and to avoid benchmarking against the wrong topology.


python3 - <<'PY'
import os,requests,json
from pathlib import Path
for line in Path('.env').read_text().splitlines():
    if '=' in line and not line.strip().startswith('#'):
        k,v=line.split('=',1);os.environ.setdefault(k.strip(),v.strip())
key=os.environ.get('VAST_KEY')
api='https://console.vast.ai/api/v0/workergroups/'
r=requests.get(api,headers={'Accept':'application/json','Authorization':f'Bearer {key}'},timeout=30)
print('status',r.status_code)
d=r.json();results=d.get('results',[]) if isinstance(d,dict) else d
for wg in results:
    print('\nWG',wg.get('id'),'endpoint_id',wg.get('endpoint_id'),'endpoint_name',wg.get('endpoint_name'),'template_hash',wg.get('template_hash'),'gpu_ram',wg.get('gpu_ram'),'active',wg.get('active'),'state',wg.get('state'))
    sp=wg.get('search_params')
    if sp: print(' search_params=',sp)
PY


python3 - <<'PY'
import os,requests,json
from pathlib import Path
for line in Path('.env').read_text().splitlines():
    if '=' in line and not line.strip().startswith('#'):
        k,v=line.split('=',1);os.environ.setdefault(k.strip(),v.strip())
key=os.environ.get('VAST_KEY')
api='https://console.vast.ai/api/v0/templates/'
r=requests.get(api,headers={'Accept':'application/json','Authorization':f'Bearer {key}'},timeout=30)
print('status',r.status_code)
d=r.json();results=d.get('results',[]) if isinstance(d,dict) else d
# print only hindi templates
for t in results:
    nm=str(t.get('name',''))
    img=str(t.get('image',''))
    if 'hindi' in nm.lower() or 'hindi' in img.lower() or 'nemotron' in img.lower():
        print('\nTEMPLATE',t.get('id'),'hash',t.get('hash'),'name',nm)
        print(' image',img)
        print(' onstart',t.get('onstart_cmd'))
        print(' env',t.get('env'))
        print(' search',t.get('search_params'))
PY


Output:
Chunk ID: 64fff2
Wall time: 0.8092 seconds
Process exited with code 0
Original token count: 74
Output:
status 200

WG 16780 endpoint_id 11967 endpoint_name hindi-asr-prod template_hash 503f03778b9c2bc36e5ae520d1625380 gpu_ram 40.0 active None state None

WG 16820 endpoint_id 11995 endpoint_name hindi-asr-ws-bench template_hash 292d1c7065d57c229af2b239b71d015f gpu_ram 40.0 active None state None