Output:
Chunk ID: 2ff917
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1959
Output:
                        "message": "Streaming pipeline not initialized.",
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

        # Non-WAV formats not implemented yet (M5)
        if req.format != AudioFormat.WAV.value:
            return JSONResponse(
                status_code=501,
                content={
                    "error": {
                        "code": "FORMAT_NOT_IMPLEMENTED",
                        "message": f"Streaming format '{req.format}' not yet implemented. Use format=wav.",
                        "request_id": request_id,
                    }
                },
                headers={"X-Request-ID": request_id},
            )

        # Set request context for structured logging
        set_request_context(request_id=request_id)

        try:
            admission_lease = await tts_runtime.acquire_streaming_slot()
        except tts_runtime.StreamingAdmissionError as exc:
            retry_after_ms = float(getattr(exc, "retry_after_ms", 0.0) or 0.0)
            retry_after_s = max(1, int((retry_after_ms + 999.0) // 1000.0))
            overload_snapshot = dict(getattr(exc, "snapshot", {}) or {})
            log_request_failed(
                request_id=request_id,
                status_code=429,
                error_code="STREAMING_OVERLOADED",
                error_message=str(exc),
            )
            metrics_request_failed(
                status_code=429,
                error_code="STREAMING_OVERLOADED",
                speaker=speaker,
            )
            clear_request_context()
            return JSONResponse(
                status_code=429,
                content={
                    "error": {
                        "code": "STREAMING_OVERLOADED",
                        "message": "Streaming capacity exhausted. Retry shortly.",
                        "request_id": request_id,
                        "reason": getattr(exc, "reason", "overloaded"),
                    },
                    "admission": overload_snapshot,
                },
                headers={
                    "X-Request-ID": request_id,
                    "Retry-After": str(retry_after_s),
                },
            )

        # Create async generator for streaming response
        async def audio_stream_generator():
            """Yields audio chunks as they're generated."""
            final_metrics = {}
            first_chunk_sent = False
            chunks_count = 0
            try:
                async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
                    text=text,
                    speaker=speaker,
                    temperature=req.temperature,
                    top_k=req.top_k,
                    top_p=req.top_p,
                    max_tokens=req.max_tokens,
                    repetition_penalty=req.repetition_penalty,
                    seed=req.seed,
                    enable_chunking=req.chunking,
                    admission_lease=admission_lease,
                    release_admission_lease=False,
                ):
                    final_metrics = metrics
                    chunks_count += 1

                    # Fire-and-forget sentence storage after first chunk (don't block TTFB)
                    if not first_chunk_sent:
                        first_chunk_sent = True
                        ttfb_ms = metrics.get("ttfb_ms", 0)

                        # Log first audio emitted (TTFB marker)
                        log_first_audio_emitted(
                            request_id=request_id,
                            ttfb_ms=ttfb_ms,
                            chunk_size_bytes=len(audio_chunk),
                        )
                        record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)

                        sentence_store = _get_sentence_store()
                        sentence_store.store_fire_and_forget(
                            request_id=request_id,
                            text=text,
                            speaker=speaker,
                            stream=True,
                            format=req.format,
                            temperature=req.temperature,
                            top_k=req.top_k,
                            top_p=req.top_p,
                            max_tokens=req.max_tokens,
                            repetition_penalty=req.repetition_penalty,
                            seed=req.seed,
                            text_chunked=metrics.get("text_chunked", False),
                            ttfb_ms=ttfb_ms,
                            # audio_duration_seconds not known yet for streaming
                        )

                    yield audio_chunk

                # Stream completed - log metrics
                total_time = time.time() - start_time
                audio_duration = final_metrics.get("audio_duration_seconds", 0)
                rtf = total_time / audio_duration if audio_duration > 0 else 0

                log_request_completed(
                    request_id=request_id,
                    status_code=200,
                    total_duration_ms=int(total_time * 1000),
                    audio_duration_seconds=audio_duration,
                    rtf=rtf,
                    chunks_sent=chunks_count,
                )
                metrics_request_completed(
                    status_code=200,
                    duration_seconds=total_time,
                    speaker=speaker,
                    stream=True,
                )
                record_rtf(rtf=rtf, speaker=speaker)
                record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
                record_chunks_sent(chunks=chunks_count, speaker=speaker)
                clear_request_context()
            except Exception as e:
                logger.exception(f"Streaming error: {e}")
                # Log streaming failure
                log_request_failed(
                    request_id=request_id,
                    status_code=500,
                    error_code="STREAMING_ERROR",
                    error_message=str(e),
                )
                metrics_request_failed(
                    status_code=500,
                    error_code="STREAMING_ERROR",
                    speaker=speaker,
                )
                clear_request_context()
                # Can't return error once streaming started
                # Best we can do is stop yielding
                return
            finally:
                tts_runtime.release_streaming_slot(admission_lease)

        # Build response headers
        # Note: Some headers can't be set dynamically during streaming
        # TTFB and RTF will be logged but not in headers for streaming
        model_version = runtime.model_version if runtime else get_model_version()

        headers = {
            "X-Request-ID": request_id,
            "X-Model-Version": model_version,
            "X-Format": "wav",
            "X-Sample-Rate": str(req.sample_rate),
            "X-Stream": "true",
            "X-Chunking-Enabled": str(req.chunking).lower(),
            "X-Admission-Wait-ms": str(int(round(float(admission_lease.wait_ms)))),
            "X-Admission-Queued": str(bool(admission_lease.queued)).lower(),
            # Transfer-Encoding: chunked is set automatically by StreamingResponse
        }

        if req.seed is not None:
            headers["X-Seed"] = str(req.seed)

        return StreamingResponse(
            audio_stream_generator(),


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '940,980p' veena3modal/api/fastapi_app.py


Output:
Chunk ID: a3cef9
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 134
Output:
            audio_stream_generator(),
            media_type="audio/wav",
            headers=headers,
            background=BackgroundTask(tts_runtime.release_streaming_slot, admission_lease),
        )

    # === WebSocket Support (M5b) ===
    try:
        from veena3modal.api.websocket_handler import add_websocket_routes
        add_websocket_routes(app)
        logger.info("WebSocket support enabled at /v1/tts/ws")
    except ImportError as e:
        logger.warning(f"WebSocket support not available: {e}")

    return app


Summary: auto


**Verifying import presence**


rg -n "BackgroundTask|def _handle_streaming_request" veena3modal/api/fastapi_app.py


Output:
Chunk ID: 8362d4
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 67
Output:
740:    async def _handle_streaming_request(req, request_id: str, text: str, speaker: str, start_time: float):
748:        from starlette.background import BackgroundTask
943:            background=BackgroundTask(tts_runtime.release_streaming_slot, admission_lease),


**Reviewing websocket patch and styles**


rg -n "acquire_streaming_slot|release_streaming_slot|STREAMING_OVERLOADED|code=1013" veena3modal/api/websocket_handler.py -n -S && sed -n '260,520p' veena3modal/api/websocket_handler.py


Output:
Chunk ID: 450079
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 2432
Output:
293:            admission_lease = await tts_runtime.acquire_streaming_slot()
298:                error_code="STREAMING_OVERLOADED",
303:                error_code="STREAMING_OVERLOADED",
307:                code="STREAMING_OVERLOADED",
311:            await websocket.close(code=1013)  # Try again later
414:            tts_runtime.release_streaming_slot(admission_lease)
484:        tts_runtime.release_streaming_slot(admission_lease)
                message=validation_error,
                request_id=request_id,
            ))
            await websocket.close(code=1003)
            return

        # Resolve speaker
        speaker = resolve_speaker_name(req.speaker)

        # Log request
        log_request_received(
            request_id=request_id,
            text_length=len(req.text),
            speaker=speaker,
            stream=True,
            format="wav",
        )
        metrics_request_received(speaker=speaker, stream=True, format="wav")

        # === CHECK MODEL ===
        if not tts_runtime.is_initialized():
            await websocket.send_text(create_error_message(
                code="MODEL_NOT_LOADED",
                message="TTS model not initialized",
                request_id=request_id,
            ))
            await websocket.close(code=1011)  # Internal error
            return

        runtime = tts_runtime.get_runtime()
        model_version = runtime.model_version if runtime else "unknown"

        try:
            admission_lease = await tts_runtime.acquire_streaming_slot()
        except tts_runtime.StreamingAdmissionError as exc:
            log_request_failed(
                request_id=request_id,
                status_code=429,
                error_code="STREAMING_OVERLOADED",
                error_message=str(exc),
            )
            metrics_request_failed(
                status_code=429,
                error_code="STREAMING_OVERLOADED",
                speaker=speaker,
            )
            await websocket.send_text(create_error_message(
                code="STREAMING_OVERLOADED",
                message="Streaming capacity exhausted. Retry shortly.",
                request_id=request_id,
            ))
            await websocket.close(code=1013)  # Try again later
            return

        # === SEND HEADER INFO ===
        await websocket.send_text(create_header_message(
            request_id=request_id,
            sample_rate=req.sample_rate,
            model_version=model_version,
        ))

        # === STREAM AUDIO ===
        start_time = time.time()
        chunks_sent = 0
        total_bytes = 0
        first_chunk_time = None
        final_metrics = {}

        # Start a task to listen for cancel messages
        async def listen_for_cancel():
            nonlocal cancelled
            try:
                while not cancelled:
                    try:
                        msg = await asyncio.wait_for(
                            websocket.receive_text(),
                            timeout=0.1
                        )
                        data = json.loads(msg)
                        if data.get("event") == "cancel":
                            cancelled = True
                            logger.info(f"Client requested cancel: {request_id}")
                            break
                        elif data.get("event") == "ping":
                            await websocket.send_text(json.dumps({"event": "pong"}))
                    except asyncio.TimeoutError:
                        continue
                    except Exception:
                        break
            except Exception:
                pass

        cancel_task = asyncio.create_task(listen_for_cancel())

        try:
            # Apply text normalization if enabled
            text = req.text
            if req.normalize:
                try:
                    from veena3modal.processing.text_normalizer import normalize_text
                    text = normalize_text(text)
                except ImportError:
                    pass  # Normalizer not available

            # Stream audio chunks
            async for audio_chunk, metrics in tts_runtime.generate_speech_streaming(
                text=text,
                speaker=speaker,
                temperature=req.temperature,
                top_k=req.top_k,
                top_p=req.top_p,
                max_tokens=req.max_tokens,
                repetition_penalty=req.repetition_penalty,
                seed=req.seed,
                enable_chunking=req.chunking,
                admission_lease=admission_lease,
                release_admission_lease=False,
            ):
                if cancelled:
                    logger.info(f"Generation cancelled: {request_id}")
                    break

                final_metrics = metrics
                chunks_sent += 1
                total_bytes += len(audio_chunk)

                # Record TTFB on first chunk
                if first_chunk_time is None:
                    first_chunk_time = time.time()
                    ttfb_ms = int((first_chunk_time - start_time) * 1000)
                    log_first_audio_emitted(
                        request_id=request_id,
                        ttfb_ms=ttfb_ms,
                        chunk_size_bytes=len(audio_chunk),
                    )
                    record_ttfb(ttfb_seconds=ttfb_ms / 1000, speaker=speaker, stream=True)

                # Send audio as binary frame
                await websocket.send_bytes(audio_chunk)

                # Send progress every 10 chunks
                if chunks_sent % 10 == 0:
                    await websocket.send_text(create_progress_message(
                        chunks_sent=chunks_sent,
                        bytes_sent=total_bytes,
                        elapsed_ms=int((time.time() - start_time) * 1000),
                    ))

        finally:
            cancel_task.cancel()
            try:
                await cancel_task
            except asyncio.CancelledError:
                pass
            tts_runtime.release_streaming_slot(admission_lease)

        # === SEND COMPLETION ===
        total_time = time.time() - start_time
        audio_duration = final_metrics.get("audio_duration_seconds", 0)
        rtf = total_time / audio_duration if audio_duration > 0 else 0

        completion_metrics = {
            "request_id": request_id,
            "chunks_sent": chunks_sent,
            "total_bytes": total_bytes,
            "audio_duration_seconds": round(audio_duration, 2),
            "total_time_seconds": round(total_time, 2),
            "ttfb_ms": int((first_chunk_time - start_time) * 1000) if first_chunk_time else 0,
            "rtf": round(rtf, 3),
            "cancelled": cancelled,
        }

        await websocket.send_text(create_complete_message(
            request_id=request_id,
            metrics=completion_metrics,
        ))

        # Log completion
        if not cancelled:
            log_request_completed(
                request_id=request_id,
                status_code=200,
                total_duration_ms=int(total_time * 1000),
                audio_duration_seconds=audio_duration,
                rtf=rtf,
                chunks_sent=chunks_sent,
            )
            metrics_request_completed(
                status_code=200,
                duration_seconds=total_time,
                speaker=speaker,
                stream=True,
            )
            record_rtf(rtf=rtf, speaker=speaker)
            record_audio_duration(duration_seconds=audio_duration, speaker=speaker)
            record_chunks_sent(chunks=chunks_sent, speaker=speaker)

        # Close connection gracefully
        await websocket.close(code=1000)

    except Exception as e:
        logger.exception(f"WebSocket error: {e}")
        log_request_failed(
            request_id=request_id,
            status_code=500,
            error_code="WEBSOCKET_ERROR",
            error_message=str(e),
        )
        metrics_request_failed(
            status_code=500,
            error_code="WEBSOCKET_ERROR",
        )

        try:
            await websocket.send_text(create_error_message(
                code="INTERNAL_ERROR",
                message=str(e),
                request_id=request_id,
            ))
            await websocket.close(code=1011)
        except Exception:
            pass

    finally:
        tts_runtime.release_streaming_slot(admission_lease)
        clear_request_context()

def add_websocket_routes(app):
    """
    Add WebSocket routes to FastAPI app.

    Call this from create_app() to enable WebSocket support.

    Args:
        app: FastAPI application instance
    """
    from fastapi import WebSocket, Query

    @app.websocket("/v1/tts/ws")
    async def websocket_tts(
        websocket: WebSocket,
        api_key: Optional[str] = Query(None, alias="api_key"),
    ):
        """
        WebSocket endpoint for TTS streaming.

        Connect with optional api_key query parameter:
        ws://host/v1/tts/ws?api_key=your-key

        Then send JSON request:
        {"text": "Hello world", "speaker": "male1"}

        Receive:
        - JSON header message
        - Binary audio chunks
        - JSON progress messages (every 10 chunks)
        - JSON completion message
        """
        # Also check Authorization header
        auth_header = websocket.headers.get("authorization", "")


**Verifying stress script patch**


rg -n "stream-admission|max_inflight|failure_reason_counts|admission_wait_ms" scripts/stress_test_streaming_runtime.py -n -S && sed -n '120,260p' scripts/stress_test_streaming_runtime.py && sed -n '300,520p' scripts/stress_test_streaming_runtime.py && sed -n '520,760p' scripts/stress_test_streaming_runtime.py


Output:
Chunk ID: b61bf3
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 6657
Output:
217:    def failure_reason_counts(self) -> Dict[str, int]:
390:        reasons = level.failure_reason_counts()
435:        "admission_wait_ms",
528:            "stream_admission_max_inflight": args.stream_admission_max_inflight,
568:                    "admission_wait_ms": lvl.timing_stats("admission_wait_ms"),
581:                "failure_reasons": lvl.failure_reason_counts(),
678:    parser.add_argument("--stream-admission-max-inflight", type=int, default=0)
679:    parser.add_argument("--stream-admission-max-queue", type=int, default=0)
680:    parser.add_argument("--stream-admission-max-wait-ms", type=float, default=0.0)
681:    parser.add_argument("--stream-admission-poll-ms", type=float, default=2.0)
706:    os.environ["VEENA3_STREAM_ADMISSION_MAX_INFLIGHT"] = str(args.stream_admission_max_inflight)
                "min": min(util),
                "avg": statistics.mean(util),
                "max": max(util),
            },
        }

@dataclass
class RequestResult:
    success: bool
    latency_ms: float
    ttfb_ms: float
    chunks_sent: int
    total_bytes: int
    audio_seconds: float
    speaker: str
    error: Optional[str] = None
    inter_chunk_ms: List[float] = field(default_factory=list)
    tail_emit_speed_x: float = 0.0
    final_metrics: Dict[str, Any] = field(default_factory=dict)

@dataclass
class LevelResult:
    concurrency: int
    total_requests: int
    results: List[RequestResult] = field(default_factory=list)
    wall_time_s: float = 0.0
    gpu_summary: Dict[str, Any] = field(default_factory=dict)

    @property
    def successes(self) -> List[RequestResult]:
        return [r for r in self.results if r.success]

    @property
    def failures(self) -> List[RequestResult]:
        return [r for r in self.results if not r.success]

    @property
    def success_rate(self) -> float:
        return len(self.successes) / len(self.results) if self.results else 0.0

    @property
    def throughput_rps(self) -> float:
        return self.total_requests / self.wall_time_s if self.wall_time_s > 0 else 0.0

    def _pct(self, vals: List[float], p: int) -> float:
        if not vals:
            return 0.0
        s = sorted(vals)
        idx = int((len(s) - 1) * (p / 100))
        return s[idx]

    def _stats(self, vals: List[float]) -> Dict[str, float]:
        if not vals:
            return {}
        return {
            "avg": statistics.mean(vals),
            "p50": self._pct(vals, 50),
            "p95": self._pct(vals, 95),
            "max": max(vals),
        }

    def latency_stats(self) -> Dict[str, float]:
        return self._stats([r.latency_ms for r in self.successes])

    def ttfb_stats(self) -> Dict[str, float]:
        return self._stats([r.ttfb_ms for r in self.successes if r.ttfb_ms > 0])

    def chunk_count_stats(self) -> Dict[str, float]:
        return self._stats([float(r.chunks_sent) for r in self.successes])

    def emit_speed_stats(self) -> Dict[str, float]:
        return self._stats([r.tail_emit_speed_x for r in self.successes if r.tail_emit_speed_x > 0])

    def inter_chunk_stats(self) -> Dict[str, float]:
        vals: List[float] = []
        for r in self.successes:
            vals.extend(r.inter_chunk_ms)
        return self._stats(vals)

    def timing_stats(self, key: str) -> Dict[str, float]:
        vals: List[float] = []
        for r in self.successes:
            v = r.final_metrics.get(key)
            if isinstance(v, (int, float)):
                vals.append(float(v))
        return self._stats(vals)

    def latency_over_metric_stats(self, key: str) -> Dict[str, float]:
        vals: List[float] = []
        for r in self.successes:
            base = r.final_metrics.get(key)
            if isinstance(base, (int, float)):
                vals.append(max(0.0, float(r.latency_ms) - float(base)))
        return self._stats(vals)

    def failure_reason_counts(self) -> Dict[str, int]:
        counts: Dict[str, int] = {}
        for r in self.failures:
            reason = (r.error or "unknown").strip().lower()
            if not reason:
                reason = "unknown"
            counts[reason] = counts.get(reason, 0) + 1
        return counts

def init_runtime(
    model_path: str,
    gpu_mem: float,
    stream_output_kind: str,
    num_engines: int,
    max_num_batched_tokens: Optional[int],
    max_num_seqs: Optional[int],
    enable_chunked_prefill: Optional[bool],
    enable_prefix_caching: Optional[bool],
    disable_log_stats: Optional[bool],
    enforce_eager: Optional[bool],
) -> None:
    from veena3modal.local_server import resolve_model_paths
    from veena3modal.services.tts_runtime import initialize_runtime, is_initialized

    os.environ["VEENA3_STREAM_OUTPUT_KIND"] = stream_output_kind

    llm_path, bicodec_path = resolve_model_paths(model_path)
    if not is_initialized():
        initialize_runtime(
            model_path=llm_path,
            bicodec_path=bicodec_path,
            device="cuda",
            gpu_memory_utilization=gpu_mem,
            num_engines=num_engines,
            max_num_batched_tokens=max_num_batched_tokens,
            max_num_seqs=max_num_seqs,
            enable_chunked_prefill=enable_chunked_prefill,
            enable_prefix_caching=enable_prefix_caching,
            disable_log_stats=disable_log_stats,
            enforce_eager=enforce_eager,
            precompute_speaker_globals=False,
        )

                # when internal stream generation completes.
                final_metrics = metrics

    try:
        await asyncio.wait_for(_consume_stream(), timeout=timeout_s)
        end = time.perf_counter()
        latency_ms = (end - t0) * 1000.0
        ttfb_ms = ((first_chunk_t - t0) * 1000.0) if first_chunk_t is not None else 0.0
        audio_seconds = float(final_metrics.get("audio_duration_seconds", 0.0) or 0.0)
        tail_s = max(1e-6, (latency_ms - ttfb_ms) / 1000.0)
        tail_emit_speed_x = audio_seconds / tail_s if audio_seconds > 0 else 0.0
        return RequestResult(
            success=True,
            latency_ms=latency_ms,
            ttfb_ms=ttfb_ms,
            chunks_sent=chunks,
            total_bytes=total_bytes,
            audio_seconds=audio_seconds,
            speaker=speaker,
            inter_chunk_ms=inter_chunk_ms,
            tail_emit_speed_x=tail_emit_speed_x,
            final_metrics=dict(final_metrics),
        )
    except asyncio.TimeoutError:
        return RequestResult(
            success=False,
            latency_ms=(time.perf_counter() - t0) * 1000.0,
            ttfb_ms=0.0,
            chunks_sent=chunks,
            total_bytes=total_bytes,
            audio_seconds=0.0,
            speaker=speaker,
            error="timeout",
            inter_chunk_ms=inter_chunk_ms,
            final_metrics=dict(final_metrics),
        )
    except Exception as exc:
        return RequestResult(
            success=False,
            latency_ms=(time.perf_counter() - t0) * 1000.0,
            ttfb_ms=0.0,
            chunks_sent=chunks,
            total_bytes=total_bytes,
            audio_seconds=0.0,
            speaker=speaker,
            error=str(exc)[:200],
            inter_chunk_ms=inter_chunk_ms,
            final_metrics=dict(final_metrics),
        )

async def run_level(
    concurrency: int,
    total_requests: int,
    text: str,
    max_tokens: int,
    timeout_s: float,
    enable_chunking: bool,
    gpu_monitor: GPUMonitor,
    seed_base: Optional[int],
) -> LevelResult:
    level = LevelResult(concurrency=concurrency, total_requests=total_requests)
    semaphore = asyncio.Semaphore(concurrency)

    async def wrapped(i: int) -> RequestResult:
        async with semaphore:
            speaker = SPEAKERS[i % len(SPEAKERS)]
            seed = (seed_base + i) if seed_base is not None else None
            return await run_one_request(
                text=text,
                speaker=speaker,
                max_tokens=max_tokens,
                timeout_s=timeout_s,
                enable_chunking=enable_chunking,
                seed=seed,
            )

    gpu_monitor.snapshots.clear()
    t0 = time.perf_counter()
    level.results = await asyncio.gather(*[wrapped(i) for i in range(total_requests)])
    level.wall_time_s = time.perf_counter() - t0
    level.gpu_summary = gpu_monitor.summary()
    return level

def print_level(level: LevelResult) -> None:
    print(f"\n=== streaming concurrency={level.concurrency} requests={level.total_requests} ===")
    print(f"success={len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"wall={level.wall_time_s:.2f}s throughput={level.throughput_rps:.2f} req/s")
    if level.failures:
        reasons = level.failure_reason_counts()
        summary = ", ".join(f"{k}:{v}" for k, v in sorted(reasons.items(), key=lambda kv: kv[1], reverse=True))
        print(f"failures: {summary}")
    lat = level.latency_stats()
    if lat:
        print(f"latency_ms avg={lat['avg']:.0f} p50={lat['p50']:.0f} p95={lat['p95']:.0f} max={lat['max']:.0f}")
    ttfb = level.ttfb_stats()
    if ttfb:
        print(f"ttfb_ms avg={ttfb['avg']:.0f} p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}")
    cc = level.chunk_count_stats()
    if cc:
        print(f"chunks_per_req avg={cc['avg']:.1f} p50={cc['p50']:.1f} p95={cc['p95']:.1f} max={cc['max']:.1f}")
    ic = level.inter_chunk_stats()
    if ic:
        print(
            "inter_chunk_ms avg={:.1f} p50={:.1f} p95={:.1f} max={:.1f}".format(
                ic["avg"], ic["p50"], ic["p95"], ic["max"]
            )
        )
    es = level.emit_speed_stats()
    if es:
        print(
            "tail_emit_speed_x avg={:.2f} p50={:.2f} p95={:.2f} max={:.2f}".format(
                es["avg"], es["p50"], es["p95"], es["max"]
            )
        )

    timing_keys = [
        "timeline_to_first_batch_ms",
        "timeline_to_first_chunk_emitted_ms",
        "timeline_total_ms",
        "llm_time_in_queue_ms",
        "llm_scheduler_ms",
        "llm_first_token_ms",
        "llm_request_lifecycle_ms",
        "llm_generation_wall_ms",
        "llm_time_per_token_ms",
        "llm_batch_wall_ms_total",
        "llm_decode_wall_ms_total",
        "tokens_per_batch_p50",
        "bicodec_decode_calls",
        "bicodec_decode_wall_ms_total",
        "bicodec_decode_interval_ms_p50",
        "decode_interval_applied_tokens_p50",
        "decode_pending_requests_p50",
        "admission_wait_ms",
        "admission_inflight_on_grant",
        "admission_waiters_on_grant",
        "admission_queue_depth_on_entry",
        "batch_avg_size",
        "batch_max_seen",
        "batch_workers_live",
        "batch_workers_target",
        "batch_queue_wait_ms_avg",
        "batch_compute_ms_avg",
        "batch_queue_depth_avg",
    ]
    for key in timing_keys:
        st = level.timing_stats(key)
        if st:
            print(f"{key}: avg={st['avg']:.2f} p50={st['p50']:.2f} p95={st['p95']:.2f} max={st['max']:.2f}")

    latency_over_timeline = level.latency_over_metric_stats("timeline_total_ms")
    if latency_over_timeline:
        print(
            "latency_over_timeline_ms: avg={:.2f} p50={:.2f} p95={:.2f} max={:.2f}".format(
                latency_over_timeline["avg"],
                latency_over_timeline["p50"],
                latency_over_timeline["p95"],
                latency_over_timeline["max"],
            )
        )

    tl_total = level.timing_stats("timeline_total_ms")
    llm_total = level.timing_stats("llm_generation_wall_ms")
    bicodec_total = level.timing_stats("bicodec_decode_wall_ms_total")
    if tl_total and llm_total:
        total_avg = tl_total.get("avg", 0.0)
        llm_avg = llm_total.get("avg", 0.0)
        bicodec_avg = bicodec_total.get("avg", 0.0) if bicodec_total else 0.0
        other_avg = max(0.0, total_avg - llm_avg - bicodec_avg)
        if total_avg > 0:
            print(
                "timeline_share: llm={:.1f}% bicodec={:.1f}% other={:.1f}%".format(
                    (llm_avg / total_avg) * 100,
                    (bicodec_avg / total_avg) * 100,
                    (other_avg / total_avg) * 100,
                )
            )

    if "memory_mb" in level.gpu_summary:
        g = level.gpu_summary
        print(
            "gpu: util_avg={:.0f}% util_max={:.0f}% mem_avg={:.0f}MB mem_max={:.0f}MB".format(
                g["gpu_util_pct"]["avg"],
                g["gpu_util_pct"]["max"],
                g["memory_mb"]["avg"],
                g["memory_mb"]["max"],
            )
        )

def save_results(path: str, levels: List[LevelResult], args: argparse.Namespace) -> None:
    payload = {
        "config": {
            "levels": args.levels,
            "text": args.text,
            "max_tokens": args.max_tokens,
            "timeout_s": args.timeout,
            "seed_base": args.seed_base,
            "requests_multiplier": args.requests_multiplier,
            "min_requests": args.min_requests,
            "gpu_memory": args.gpu_memory,
            "stream_output_kind": args.stream_output_kind,
            "chunking": args.chunking,
            "num_engines": args.num_engines,
            "max_num_batched_tokens": args.max_num_batched_tokens,
            "max_num_seqs": args.max_num_seqs,
            "enable_chunked_prefill": args.enable_chunked_prefill,
            "disable_chunked_prefill": args.disable_chunked_prefill,
            "disable_prefix_caching": args.disable_prefix_caching,
            "disable_engine_stats_logs": args.disable_engine_stats_logs,
            "enforce_eager": args.enforce_eager,
            "stream_decode_interval": args.stream_decode_interval,
            "stream_window_size": args.stream_window_size,
            "stream_min_semantic_first": args.stream_min_semantic_first,
            "stream_crossfade_ms": args.stream_crossfade_ms,
            "adaptive_decode": args.adaptive_decode,
            "adaptive_decode_first_interval": args.adaptive_decode_first_interval,
            "adaptive_decode_busy_interval": args.adaptive_decode_busy_interval,
            "adaptive_decode_busy_pending": args.adaptive_decode_busy_pending,
            "adaptive_decode_busy_pending": args.adaptive_decode_busy_pending,
            "disable_windowed_decode": args.disable_windowed_decode,
            "disable_bicodec_batching": args.disable_bicodec_batching,
            "bicodec_batch_max": args.bicodec_batch_max,
            "bicodec_batch_timeout_ms": args.bicodec_batch_timeout_ms,
            "bicodec_batch_workers": args.bicodec_batch_workers,
            "bicodec_batch_scale_pending": args.bicodec_batch_scale_pending,
            "bicodec_batch_scale_mode": args.bicodec_batch_scale_mode,
            "stream_admission_max_inflight": args.stream_admission_max_inflight,
            "stream_admission_max_queue": args.stream_admission_max_queue,
            "stream_admission_max_wait_ms": args.stream_admission_max_wait_ms,
            "stream_admission_poll_ms": args.stream_admission_poll_ms,
            "timestamp": int(time.time()),
        },
        "levels": [],
    }
    for lvl in levels:
        payload["levels"].append(
            {
                "concurrency": lvl.concurrency,
                "total_requests": lvl.total_requests,
                "success_rate": lvl.success_rate,
                "throughput_rps": lvl.throughput_rps,
                "wall_time_s": lvl.wall_time_s,
                "latency_ms": lvl.latency_stats(),
                "ttfb_ms": lvl.ttfb_stats(),
                "chunks_per_req": lvl.chunk_count_stats(),
                "inter_chunk_ms": lvl.inter_chunk_stats(),
                "tail_emit_speed_x": lvl.emit_speed_stats(),
                "gpu": lvl.gpu_summary,
                "timing": {
                    "timeline_to_first_batch_ms": lvl.timing_stats("timeline_to_first_batch_ms"),
                    "timeline_to_first_chunk_emitted_ms": lvl.timing_stats("timeline_to_first_chunk_emitted_ms"),
                    "timeline_total_ms": lvl.timing_stats("timeline_total_ms"),
                    "llm_time_in_queue_ms": lvl.timing_stats("llm_time_in_queue_ms"),
                    "llm_scheduler_ms": lvl.timing_stats("llm_scheduler_ms"),
                    "llm_first_token_ms": lvl.timing_stats("llm_first_token_ms"),
                    "llm_request_lifecycle_ms": lvl.timing_stats("llm_request_lifecycle_ms"),
                    "llm_generation_wall_ms": lvl.timing_stats("llm_generation_wall_ms"),
                    "llm_time_per_token_ms": lvl.timing_stats("llm_time_per_token_ms"),
                    "llm_batch_wall_ms_total": lvl.timing_stats("llm_batch_wall_ms_total"),
                    "llm_decode_wall_ms_total": lvl.timing_stats("llm_decode_wall_ms_total"),
                    "tokens_per_batch_p50": lvl.timing_stats("tokens_per_batch_p50"),
                    "bicodec_decode_calls": lvl.timing_stats("bicodec_decode_calls"),
                    "bicodec_decode_wall_ms_total": lvl.timing_stats("bicodec_decode_wall_ms_total"),
                    "bicodec_decode_interval_ms_p50": lvl.timing_stats("bicodec_decode_interval_ms_p50"),
                    "decode_interval_applied_tokens_p50": lvl.timing_stats("decode_interval_applied_tokens_p50"),
                    "decode_pending_requests_p50": lvl.timing_stats("decode_pending_requests_p50"),
                    "admission_wait_ms": lvl.timing_stats("admission_wait_ms"),
                    "admission_inflight_on_grant": lvl.timing_stats("admission_inflight_on_grant"),
                    "admission_waiters_on_grant": lvl.timing_stats("admission_waiters_on_grant"),
                    "admission_queue_depth_on_entry": lvl.timing_stats("admission_queue_depth_on_entry"),
                    "batch_avg_size": lvl.timing_stats("batch_avg_size"),
                    "batch_max_seen": lvl.timing_stats("batch_max_seen"),
                    "batch_workers_live": lvl.timing_stats("batch_workers_live"),
                    "batch_workers_target": lvl.timing_stats("batch_workers_target"),
                    "batch_queue_wait_ms_avg": lvl.timing_stats("batch_queue_wait_ms_avg"),
                    "batch_compute_ms_avg": lvl.timing_stats("batch_compute_ms_avg"),
                    "batch_queue_depth_avg": lvl.timing_stats("batch_queue_depth_avg"),
                    "latency_over_timeline_ms": lvl.latency_over_metric_stats("timeline_total_ms"),
                },
                "failure_reasons": lvl.failure_reason_counts(),
                "requests": [
                    {
                        "success": r.success,
                        "latency_ms": r.latency_ms,
                        "ttfb_ms": r.ttfb_ms,
                        "chunks_sent": r.chunks_sent,
                        "total_bytes": r.total_bytes,
                        "audio_seconds": r.audio_seconds,
                        "speaker": r.speaker,
                        "error": r.error,
                        "inter_chunk_ms": r.inter_chunk_ms,
                        "tail_emit_speed_x": r.tail_emit_speed_x,
                        "final_metrics": r.final_metrics,
                    }
                    for r in lvl.results
                ],
            }
        )
    with open(path, "w", encoding="utf-8") as fd:
        json.dump(payload, fd, indent=2)

async def main_async(args: argparse.Namespace) -> None:
    text = TEST_TEXTS[args.text]
    levels = [int(x.strip()) for x in args.levels.split(",") if x.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    # warmup
    await run_one_request(
        text=text,
        speaker=SPEAKERS[0],
        max_tokens=args.max_tokens,
        timeout_s=args.timeout,
        enable_chunking=args.chunking,
        seed=args.seed_base,
    )

    results: List[LevelResult] = []
    for idx, c in enumerate(levels):
        n = max(args.min_requests, c * args.requests_multiplier)
        level = await run_level(
            concurrency=c,
            total_requests=n,
            text=text,
            max_tokens=args.max_tokens,
            timeout_s=args.timeout,
            enable_chunking=args.chunking,
            gpu_monitor=gpu_monitor,
            seed_base=args.seed_base,
        )
        print_level(level)
        results.append(level)
        if idx < len(levels) - 1:
            await asyncio.sleep(2)

    gpu_monitor.stop()
    save_results(args.output, results, args)
    print(f"\nresults saved: {args.output}")

def main() -> None:
    parser = argparse.ArgumentParser(description="Direct runtime streaming stress benchmark")
    parser.add_argument("--levels", default="16,32,64", help="Comma-separated concurrency levels")
    parser.add_argument("--text", choices=["short", "medium", "long"], default="short")
    parser.add_argument("--max-tokens", type=int, default=512, help="max_tokens sent to streaming generation")
    parser.add_argument("--seed-base", type=int, default=1337, help="Base random seed; request i uses seed_base+i")
    parser.add_argument("--timeout", type=float, default=180.0, help="Per-request timeout seconds")
    parser.add_argument("--requests-multiplier", type=int, default=2, help="Requests per level ~= concurrency * multiplier")
    parser.add_argument("--min-requests", type=int, default=8)
    parser.add_argument("--chunking", action="store_true", default=False, help="Enable long-text chunking path")
    parser.add_argument("--gpu-memory", type=float, default=0.25)
    parser.add_argument("--stream-output-kind", choices=["delta", "cumulative"], default="delta")
    parser.add_argument("--num-engines", type=int, default=1, help="Number of vLLM engines")
    parser.add_argument("--max-num-batched-tokens", type=int, default=None)
    parser.add_argument("--max-num-seqs", type=int, default=None)
    parser.add_argument("--enable-chunked-prefill", action="store_true", default=None)
    parser.add_argument("--disable-chunked-prefill", action="store_true", default=False)
    parser.add_argument("--disable-prefix-caching", action="store_true", default=False)
    parser.add_argument("--disable-engine-stats-logs", action="store_true", default=False)
    parser.add_argument("--enforce-eager", action="store_true", default=False)
    parser.add_argument("--stream-decode-interval", type=int, default=48)
    parser.add_argument("--stream-window-size", type=int, default=128)
    parser.add_argument("--stream-min-semantic-first", type=int, default=10)
    parser.add_argument("--stream-crossfade-ms", type=int, default=50)
    parser.add_argument("--adaptive-decode", action="store_true", default=False)
    parser.add_argument("--adaptive-decode-first-interval", type=int, default=48)
    parser.add_argument("--adaptive-decode-busy-interval", type=int, default=64)
    parser.add_argument("--adaptive-decode-busy-pending", type=int, default=32)
    parser.add_argument("--disable-windowed-decode", action="store_true", default=False)
    parser.add_argument("--disable-bicodec-batching", action="store_true", default=False)
    parser.add_argument("--bicodec-batch-max", type=int, default=16)
    parser.add_argument("--bicodec-batch-timeout-ms", type=float, default=6.0)
    parser.add_argument("--bicodec-batch-workers", type=int, default=1)
    parser.add_argument("--bicodec-batch-scale-pending", type=int, default=0)
    parser.add_argument("--bicodec-batch-scale-mode", choices=["sticky", "dynamic"], default="sticky")
    parser.add_argument("--stream-admission-max-inflight", type=int, default=0)
    parser.add_argument("--stream-admission-max-queue", type=int, default=0)
    parser.add_argument("--stream-admission-max-wait-ms", type=float, default=0.0)
    parser.add_argument("--stream-admission-poll-ms", type=float, default=2.0)
    parser.add_argument("--model-path", type=str, default="")
    parser.add_argument("--output", default="stress_streaming_runtime.json")
    args = parser.parse_args()

    from veena3modal.local_server import DEFAULT_LOCAL_MODEL_DIR

    model_path = args.model_path or DEFAULT_LOCAL_MODEL_DIR

    os.environ["VEENA3_STREAM_OUTPUT_KIND"] = args.stream_output_kind
    os.environ["VEENA3_STREAM_DECODE_INTERVAL"] = str(args.stream_decode_interval)
    os.environ["VEENA3_STREAM_WINDOW_SIZE"] = str(args.stream_window_size)
    os.environ["VEENA3_STREAM_MIN_SEMANTIC_FIRST"] = str(args.stream_min_semantic_first)
    os.environ["VEENA3_STREAM_CROSSFADE_MS"] = str(args.stream_crossfade_ms)
    os.environ["VEENA3_STREAM_ADAPTIVE_DECODE"] = "1" if args.adaptive_decode else "0"
    os.environ["VEENA3_STREAM_DECODE_INTERVAL_FIRST"] = str(args.adaptive_decode_first_interval)
    os.environ["VEENA3_STREAM_DECODE_INTERVAL_BUSY"] = str(args.adaptive_decode_busy_interval)
    os.environ["VEENA3_STREAM_DECODE_BUSY_PENDING"] = str(args.adaptive_decode_busy_pending)
    os.environ["VEENA3_STREAM_WINDOWED_DECODE"] = "0" if args.disable_windowed_decode else "1"
    os.environ["VEENA3_BICODEC_BATCHING"] = "0" if args.disable_bicodec_batching else "1"
    os.environ["VEENA3_BICODEC_BATCH_MAX"] = str(args.bicodec_batch_max)
    os.environ["VEENA3_BICODEC_BATCH_TIMEOUT_MS"] = str(args.bicodec_batch_timeout_ms)
    os.environ["VEENA3_BICODEC_BATCH_WORKERS"] = str(args.bicodec_batch_workers)
    os.environ["VEENA3_BICODEC_BATCH_SCALE_PENDING"] = str(args.bicodec_batch_scale_pending)
    os.environ["VEENA3_BICODEC_BATCH_SCALE_MODE"] = str(args.bicodec_batch_scale_mode)
    os.environ["VEENA3_STREAM_ADMISSION_MAX_INFLIGHT"] = str(args.stream_admission_max_inflight)
    os.environ["VEENA3_STREAM_ADMISSION_MAX_QUEUE"] = str(args.stream_admission_max_queue)
    os.environ["VEENA3_STREAM_ADMISSION_MAX_WAIT_MS"] = str(args.stream_admission_max_wait_ms)
    os.environ["VEENA3_STREAM_ADMISSION_POLL_MS"] = str(args.stream_admission_poll_ms)

    enable_chunked_prefill = args.enable_chunked_prefill
    if args.disable_chunked_prefill:
        enable_chunked_prefill = False

    init_runtime(
        model_path=model_path,
        gpu_mem=args.gpu_memory,
        stream_output_kind=args.stream_output_kind,
        num_engines=args.num_engines,
        max_num_batched_tokens=args.max_num_batched_tokens,
        max_num_seqs=args.max_num_seqs,
        enable_chunked_prefill=enable_chunked_prefill,
        enable_prefix_caching=(False if args.disable_prefix_caching else None),
        disable_log_stats=(True if args.disable_engine_stats_logs else None),
        enforce_eager=(True if args.enforce_eager else None),
    )
    asyncio.run(main_async(args))

if __name__ == "__main__":
    main()


**Noticing existing duplicate config key**


Compile checks passed with the new admission controller and route wiring. I’m starting benchmark runs now: first the current best decode profile without admission caps, then the same profile with capped inflight+queue+wait so we can measure throughput vs tail-latency/rejection tradeoff.


Output:
Chunk ID: 3d6c92
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 10
Output:
NVIDIA A100-SXM4-80GB, 81920, 21197, 0


**Planning detailed benchmark runs**


python3 scripts/stress_test_streaming_runtime.py --levels 96,128 --text short --max-tokens 1024 --seed-base 2026 --requests-multiplier 6 --min-requests 256 --stream-decode-interval 48 --stream-window-size 128 --stream-min-semantic-first 10 --stream-crossfade-ms 50 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --bicodec-batch-workers 2 --bicodec-batch-scale-pending 56 --bicodec-batch-scale-mode dynamic --output /tmp/stream_admission_off.json


Output:
Chunk ID: 604aca
Wall time: 1.0011 seconds
Process running with session ID 31491
Original token count: 0
Output:


**Planning interactive polling strategy**


Baseline run is in progress on the dynamic decode profile (no admission caps). I’m polling output and will capture full level metrics and result JSON once both concurrency levels finish.


Output:
Chunk ID: dc2557
Wall time: 5.0015 seconds
Process running with session ID 31491
Original token count: 1400
Output:
[2026-02-19 04:16:12] INFO tts_runtime.py:408: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.25
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-19 04:16:12 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-19 04:16:12 [model.py:1561] Using max model len 4096
INFO 02-19 04:16:12 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-19 04:16:12 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:13 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:16 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:35949 backend=nccl
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:16 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:16 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...


**Scheduling periodic update check**


Model warmup is still underway for this run; once initialization completes I’ll capture both concurrency levels and summarize latency/throughput deltas.


Output:
Chunk ID: bded24
Wall time: 5.0012 seconds
Process running with session ID 31491
Original token count: 384
Output:
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:17 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=578512) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=578512) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=578512)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=578512)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.43s/it]
(EngineCore_DP0 pid=578512)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.43s/it]
(EngineCore_DP0 pid=578512)
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:19 [default_loader.py:291] Loading weights took 1.51 seconds
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:19 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 2.059909 seconds
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:24 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:24 [backends.py:872] Dynamo bytecode transform time: 4.19 s


**Extending polling for long compile**


Output:
Chunk ID: 3fdad9
Wall time: 5.0008 seconds
Process running with session ID 31491
Original token count: 785
Output:
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:27 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.630 s
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:27 [monitor.py:34] torch.compile takes 4.82 s in total
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:28 [gpu_worker.py:356] Available KV cache memory: 18.06 GiB
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:28 [kv_cache_utils.py:1307] GPU KV cache size: 1,577,712 tokens
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:28 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 385.18x
(EngineCore_DP0 pid=578512)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 33.34it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 35.69it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 36.67it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 37.11it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 37.25it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 37.17it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 36.87it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 34.93it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:00<00:00, 35.06it/s]
(EngineCore_DP0 pid=578512)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 28.09it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 32.44it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 33.82it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 34.60it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.34it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 34.30it/s]
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:30 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=578512) INFO 02-19 04:16:30 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.74 seconds
[2026-02-19 04:16:32] INFO tts_runtime.py:417: Initializing prompt builder...
[2026-02-19 04:16:32] INFO tts_runtime.py:424: Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


**Continuing benchmark polling**


Output:
Chunk ID: ac1bbb
Wall time: 5.0010 seconds
Process running with session ID 31491
Original token count: 147
Output:
[2026-02-19 04:16:38] INFO tts_runtime.py:434: Initializing TTS pipeline...
[2026-02-19 04:16:38] INFO tts_runtime.py:445: Initializing streaming pipeline...
[2026-02-19 04:16:38] INFO long_text_processor.py:112: LongTextProcessor initialized
[2026-02-19 04:16:38] INFO tts_runtime.py:504: ✅ TTS runtime initialized in 32025ms
[2026-02-19 04:16:38] INFO tts_runtime.py:505:    Model version: LLM
[2026-02-19 04:16:38] INFO tts_runtime.py:515:    Streaming admission: disabled
[2026-02-19 04:16:38] INFO tts_runtime.py:526: Speaker globals cache startup precompute skipped (disabled)


Summary: auto


**Planning benchmark polling**


Initialization completed; benchmark levels are now executing. I’ll report the baseline numbers once the run finishes and then launch the capped-admission run with the same decode settings.


Output:
Chunk ID: f83416
Wall time: 5.0003 seconds
Process running with session ID 31491
Original token count: 0
Output:


Output:
Chunk ID: 2b9241
Wall time: 5.0015 seconds
Process running with session ID 31491
Original token count: 0
Output:


Output:
Chunk ID: 9b6a6e
Wall time: 5.0003 seconds
Process running with session ID 31491
Original token count: 0
Output:


**Sending periodic update**


The first high-concurrency level is still executing (no failures surfaced yet). I’m waiting for the per-level summary block before proceeding to the admission-capped A/B run.


Output:
Chunk ID: 870dd0
Wall time: 5.0014 seconds
Process running with session ID 31491
Original token count: 0
Output:


**Assessing potential deadlock delay**


Output:
Chunk ID: 1e3b12
Wall time: 5.0011 seconds
Process running with session ID 31491
Original token count: 0
Output:


**Assessing output buffering issue**


Output:
Chunk ID: 893597
Wall time: 5.3101 seconds
Process exited with code 0
Original token count: 1376
Output:
✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cuda
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
   ℹ  Micro-batching enabled (workers=2, scale_mode=dynamic, scale_pending=56, max_batch=16, timeout_ms=6.0)
🚀 SparkTTSPipeline initialized
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)

=== streaming concurrency=96 requests=576 ===
success=576/576 (100%)
wall=23.54s throughput=24.47 req/s
latency_ms avg=3721 p50=3763 p95=4539 max=5123
ttfb_ms avg=1788 p50=1842 p95=2490 max=2882
chunks_per_req avg=4.5 p50=5.0 p95=5.0 max=6.0
inter_chunk_ms avg=540.6 p50=551.6 p95=1590.7 max=2244.8
tail_emit_speed_x avg=2.42 p50=2.00 p95=5.17 max=10.60
timeline_to_first_batch_ms: avg=63.52 p50=39.58 p95=143.89 max=845.17
timeline_to_first_chunk_emitted_ms: avg=1760.07 p50=1821.03 p95=2454.47 max=2817.41
timeline_total_ms: avg=3641.02 p50=3656.22 p95=4529.07 max=5044.70
llm_time_in_queue_ms: avg=0.03 p50=0.02 p95=0.14 max=0.28
llm_scheduler_ms: avg=25.51 p50=23.30 p95=44.64 max=53.17
llm_first_token_ms: avg=56.53 p50=36.75 p95=132.40 max=522.59
llm_request_lifecycle_ms: avg=2646.45 p50=2658.59 p95=3083.65 max=3738.19
llm_generation_wall_ms: avg=3623.58 p50=3623.74 p95=4508.94 max=4994.95
llm_time_per_token_ms: avg=16.06 p50=16.12 p95=19.99 max=22.92
llm_batch_wall_ms_total: avg=3231.11 p50=3193.97 p95=3995.10 max=4994.89
llm_decode_wall_ms_total: avg=3167.63 p50=3152.01 p95=3966.43 max=4959.21
tokens_per_batch_p50: avg=1.42 p50=1.00 p95=1.00 max=81.00
bicodec_decode_calls: avg=3.48 p50=4.00 p95=4.00 max=5.00
bicodec_decode_wall_ms_total: avg=2355.93 p50=2460.73 p95=3307.00 max=4010.37
bicodec_decode_interval_ms_p50: avg=656.97 p50=687.72 p95=1201.87 max=1472.00
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
admission_wait_ms: avg=0.00 p50=0.00 p95=0.00 max=0.00
admission_inflight_on_grant: avg=83.16 p50=90.00 p95=96.00 max=96.00
admission_waiters_on_grant: avg=0.00 p50=0.00 p95=0.00 max=0.00
admission_queue_depth_on_entry: avg=0.00 p50=0.00 p95=0.00 max=0.00
batch_avg_size: avg=8.96 p50=8.19 p95=12.43 max=12.95
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_workers_live: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_workers_target: avg=1.74 p50=2.00 p95=2.00 max=2.00
batch_queue_wait_ms_avg: avg=498.90 p50=511.23 p95=595.53 max=600.74
batch_compute_ms_avg: avg=157.03 p50=147.14 p95=204.33 max=220.62
batch_queue_depth_avg: avg=43.66 p50=46.01 p95=47.89 max=48.13
latency_over_timeline_ms: avg=80.35 p50=49.42 p95=289.05 max=533.56
timeline_share: llm=99.5% bicodec=64.7% other=0.0%
gpu: util_avg=91% util_max=100% mem_avg=49260MB mem_max=50025MB

=== streaming concurrency=128 requests=768 ===
success=768/768 (100%)
wall=28.00s throughput=27.43 req/s
latency_ms avg=4482 p50=4383 p95=5627 max=7402
ttfb_ms avg=2197 p50=2264 p95=3012 max=3564
chunks_per_req avg=4.3 p50=4.0 p95=5.0 max=6.0
inter_chunk_ms avg=677.0 p50=736.3 p95=1874.3 max=2901.9
tail_emit_speed_x avg=2.05 p50=1.65 p95=4.21 max=6.96
timeline_to_first_batch_ms: avg=58.65 p50=44.00 p95=124.88 max=610.23
timeline_to_first_chunk_emitted_ms: avg=2168.04 p50=2240.76 p95=2996.51 max=3561.09
timeline_total_ms: avg=4400.13 p50=4296.47 p95=5553.85 max=7369.68
llm_time_in_queue_ms: avg=0.05 p50=0.02 p95=0.25 max=0.44
llm_scheduler_ms: avg=29.02 p50=26.26 p95=40.54 max=66.82
llm_first_token_ms: avg=51.76 p50=39.88 p95=111.57 max=499.17
llm_request_lifecycle_ms: avg=3059.45 p50=3062.98 p95=3557.60 max=4729.20
llm_generation_wall_ms: avg=4386.04 p50=4280.95 p95=5552.20 max=7369.64
llm_time_per_token_ms: avg=19.47 p50=19.33 p95=24.85 max=27.81
llm_batch_wall_ms_total: avg=3795.39 p50=3679.42 p95=5003.67 max=6280.17
llm_decode_wall_ms_total: avg=3736.78 p50=3598.68 p95=4960.08 max=6249.06
tokens_per_batch_p50: avg=1.00 p50=1.00 p95=1.00 max=1.00
bicodec_decode_calls: avg=3.30 p50=3.00 p95=4.00 max=5.00
bicodec_decode_wall_ms_total: avg=3052.71 p50=3089.26 p95=4273.42 max=6286.00
bicodec_decode_interval_ms_p50: avg=805.00 p50=901.23 p95=1395.61 max=1760.59
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
admission_wait_ms: avg=0.00 p50=0.00 p95=0.00 max=0.01
admission_inflight_on_grant: avg=111.50 p50=121.00 p95=128.00 max=128.00
admission_waiters_on_grant: avg=0.00 p50=0.00 p95=0.00 max=0.00
admission_queue_depth_on_entry: avg=0.00 p50=0.00 p95=0.00 max=0.00
batch_avg_size: avg=7.83 p50=7.82 p95=8.12 max=8.19
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_workers_live: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_workers_target: avg=1.86 p50=2.00 p95=2.00 max=2.00
batch_queue_wait_ms_avg: avg=677.75 p50=699.39 p95=750.91 max=755.69
batch_compute_ms_avg: avg=137.81 p50=137.69 p95=139.62 max=140.39
batch_queue_depth_avg: avg=55.48 p50=56.86 p95=60.87 max=60.98
latency_over_timeline_ms: avg=82.36 p50=56.41 p95=214.36 max=549.71
timeline_share: llm=99.7% bicodec=69.4% other=0.0%
gpu: util_avg=91% util_max=99% mem_avg=50915MB mem_max=50969MB

results saved: /tmp/stream_admission_off.json
ERROR 02-19 04:17:34 [core_client.py:605] Engine core proc EngineCore_DP0 died unexpectedly, shutting down client.


**Planning admission control runs**


Baseline completed and saved. I’m now running the same workload with strict admission control (`max_inflight=96`, bounded queue, short wait budget) to quantify tail reduction versus rejected overflow.


python3 scripts/stress_test_streaming_runtime.py --levels 96,128 --text short --max-tokens 1024 --seed-base 2026 --requests-multiplier 6 --min-requests 256 --stream-decode-interval 48 --stream-window-size 128 --stream-min-semantic-first 10 --stream-crossfade-ms 50 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --bicodec-batch-workers 2 --bicodec-batch-scale-pending 56 --bicodec-batch-scale-mode dynamic --stream-admission-max-inflight 96 --stream-admission-max-queue 32 --stream-admission-max-wait-ms 250 --stream-admission-poll-ms 2 --output /tmp/stream_admission_cap96.json


Output:
Chunk ID: 1a672e
Wall time: 1.0011 seconds
Process running with session ID 84093
Original token count: 0
Output:


Output:
Chunk ID: e1e397
Wall time: 5.0018 seconds
Process running with session ID 84093
Original token count: 1277
Output:
[2026-02-19 04:17:52] INFO tts_runtime.py:408: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.25
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-19 04:17:53 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-19 04:17:53 [model.py:1561] Using max model len 4096
INFO 02-19 04:17:53 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-19 04:17:53 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=580257) INFO 02-19 04:17:54 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}


**Continuing polls processing**


Output:
Chunk ID: 6b4d44
Wall time: 5.0006 seconds
Process running with session ID 84093
Original token count: 434
Output:
(EngineCore_DP0 pid=580257) INFO 02-19 04:17:56 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:35001 backend=nccl
(EngineCore_DP0 pid=580257) INFO 02-19 04:17:56 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=580257) INFO 02-19 04:17:57 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=580257) INFO 02-19 04:17:57 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=580257) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=580257) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=580257)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=580257)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.15it/s]
(EngineCore_DP0 pid=580257)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.15it/s]
(EngineCore_DP0 pid=580257)
(EngineCore_DP0 pid=580257) INFO 02-19 04:17:58 [default_loader.py:291] Loading weights took 0.95 seconds
(EngineCore_DP0 pid=580257) INFO 02-19 04:17:59 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.479703 seconds


**Planning periodic progress updates**


The capped run is active and currently in model warmup; once initialized I’ll extract failure reason breakdowns (queue-full vs timeout) plus latency/tail changes against baseline.


Output:
Chunk ID: 66eca6
Wall time: 5.0032 seconds
Process running with session ID 84093
Original token count: 798
Output:
(EngineCore_DP0 pid=580257) INFO 02-19 04:18:04 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=580257) INFO 02-19 04:18:04 [backends.py:872] Dynamo bytecode transform time: 4.22 s
(EngineCore_DP0 pid=580257) INFO 02-19 04:18:07 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.687 s
(EngineCore_DP0 pid=580257) INFO 02-19 04:18:07 [monitor.py:34] torch.compile takes 4.90 s in total
(EngineCore_DP0 pid=580257) INFO 02-19 04:18:08 [gpu_worker.py:356] Available KV cache memory: 18.06 GiB
(EngineCore_DP0 pid=580257) INFO 02-19 04:18:08 [kv_cache_utils.py:1307] GPU KV cache size: 1,577,712 tokens
(EngineCore_DP0 pid=580257) INFO 02-19 04:18:08 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 385.18x
(EngineCore_DP0 pid=580257)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 31.89it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 34.77it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 35.26it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 35.60it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 35.81it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 35.89it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 35.67it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 34.09it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:01<00:00, 34.12it/s]
(EngineCore_DP0 pid=580257)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 27.98it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 32.49it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 33.76it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 34.39it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 34.90it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 34.01it/s]
(EngineCore_DP0 pid=580257) INFO 02-19 04:18:10 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=580257) INFO 02-19 04:18:10 [core.py:272] init engine (profile, create kv cache, warmup model) took 11.11 seconds
[2026-02-19 04:18:12] INFO tts_runtime.py:417: Initializing prompt builder...
[2026-02-19 04:18:12] INFO tts_runtime.py:424: Initializing BiCodec decoder...


Output:
Chunk ID: 1e3377
Wall time: 5.0006 seconds
Process running with session ID 84093
Original token count: 216
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)
[2026-02-19 04:18:18] INFO tts_runtime.py:434: Initializing TTS pipeline...
[2026-02-19 04:18:18] INFO tts_runtime.py:445: Initializing streaming pipeline...
[2026-02-19 04:18:19] INFO long_text_processor.py:112: LongTextProcessor initialized
[2026-02-19 04:18:19] INFO tts_runtime.py:504: ✅ TTS runtime initialized in 32098ms
[2026-02-19 04:18:19] INFO tts_runtime.py:505:    Model version: LLM
[2026-02-19 04:18:19] INFO tts_runtime.py:507:    Streaming admission: inflight<=96 queue<=32 wait<=250.0ms poll=2.0ms
[2026-02-19 04:18:19] INFO tts_runtime.py:526: Speaker globals cache startup precompute skipped (disabled)


**Running benchmark with polling**


Output:
Chunk ID: 3d2986
Wall time: 5.0010 seconds
Process running with session ID 84093
Original token count: 0
Output:


**Adjusting output buffering strategy**


Output:
Chunk ID: a716bf
Wall time: 24.9021 seconds
Process exited with code 0
Original token count: 1389
Output:
✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cuda
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
   ℹ  Micro-batching enabled (workers=2, scale_mode=dynamic, scale_pending=56, max_batch=16, timeout_ms=6.0)
🚀 SparkTTSPipeline initialized
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)

=== streaming concurrency=96 requests=576 ===
success=576/576 (100%)
wall=23.21s throughput=24.82 req/s
latency_ms avg=3611 p50=3506 p95=4635 max=5282
ttfb_ms avg=1809 p50=1807 p95=2586 max=3095
chunks_per_req avg=4.5 p50=5.0 p95=5.0 max=6.0
inter_chunk_ms avg=501.8 p50=454.1 p95=1480.2 max=2175.0
tail_emit_speed_x avg=2.53 p50=2.30 p95=4.70 max=7.76
timeline_to_first_batch_ms: avg=69.23 p50=42.34 p95=246.59 max=835.49
timeline_to_first_chunk_emitted_ms: avg=1776.55 p50=1753.64 p95=2575.33 max=3047.17
timeline_total_ms: avg=3522.30 p50=3413.20 p95=4568.55 max=5173.87
llm_time_in_queue_ms: avg=0.04 p50=0.02 p95=0.20 max=0.34
llm_scheduler_ms: avg=26.84 p50=25.48 p95=52.19 max=52.19
llm_first_token_ms: avg=58.36 p50=39.87 p95=159.91 max=384.38
llm_request_lifecycle_ms: avg=2632.81 p50=2623.80 p95=3085.07 max=3447.44
llm_generation_wall_ms: avg=3504.95 p50=3392.73 p95=4568.51 max=5173.82
llm_time_per_token_ms: avg=15.54 p50=15.37 p95=19.57 max=22.00
llm_batch_wall_ms_total: avg=3167.60 p50=3151.27 p95=3918.67 max=4664.00
llm_decode_wall_ms_total: avg=3098.42 p50=3096.12 p95=3811.24 max=4619.90
tokens_per_batch_p50: avg=1.27 p50=1.00 p95=1.00 max=80.00
bicodec_decode_calls: avg=3.48 p50=4.00 p95=4.00 max=5.00
bicodec_decode_wall_ms_total: avg=2308.83 p50=2285.45 p95=3467.74 max=3902.51
bicodec_decode_interval_ms_p50: avg=579.04 p50=611.24 p95=1106.62 max=1567.89
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
admission_wait_ms: avg=0.00 p50=0.00 p95=0.00 max=0.01
admission_inflight_on_grant: avg=82.85 p50=90.00 p95=96.00 max=96.00
admission_waiters_on_grant: avg=0.00 p50=0.00 p95=0.00 max=0.00
admission_queue_depth_on_entry: avg=0.00 p50=0.00 p95=0.00 max=0.00
batch_avg_size: avg=8.51 p50=8.25 p95=11.41 max=12.35
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_workers_live: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_workers_target: avg=1.66 p50=2.00 p95=2.00 max=2.00
batch_queue_wait_ms_avg: avg=545.09 p50=580.07 p95=625.24 max=635.18
batch_compute_ms_avg: avg=143.39 p50=138.21 p95=188.62 max=197.03
batch_queue_depth_avg: avg=47.01 p50=48.25 p95=50.85 max=50.90
latency_over_timeline_ms: avg=88.90 p50=54.36 p95=272.16 max=498.95
timeline_share: llm=99.5% bicodec=65.5% other=0.0%
gpu: util_avg=90% util_max=99% mem_avg=49284MB mem_max=50397MB

=== streaming concurrency=128 requests=768 ===
success=212/768 (28%)
wall=8.71s throughput=88.19 req/s
failures: streaming admission wait timeout:556
latency_ms avg=3593 p50=3612 p95=4722 max=4787
ttfb_ms avg=1533 p50=1614 p95=2358 max=2561
chunks_per_req avg=4.8 p50=5.0 p95=6.0 max=6.0
inter_chunk_ms avg=536.0 p50=563.1 p95=1620.8 max=2112.0
tail_emit_speed_x avg=2.32 p50=2.02 p95=5.65 max=9.01
timeline_to_first_batch_ms: avg=68.66 p50=53.66 p95=170.24 max=383.76
timeline_to_first_chunk_emitted_ms: avg=1446.30 p50=1453.39 p95=2209.99 max=2429.83
timeline_total_ms: avg=3468.79 p50=3426.59 p95=4684.14 max=4747.08
llm_time_in_queue_ms: avg=0.11 p50=0.03 p95=0.46 max=0.53
llm_scheduler_ms: avg=29.25 p50=29.28 p95=48.14 max=48.14
llm_first_token_ms: avg=59.62 p50=50.62 p95=146.57 max=229.81
llm_request_lifecycle_ms: avg=2559.08 p50=2540.71 p95=2971.41 max=3306.50
llm_generation_wall_ms: avg=3452.20 p50=3422.84 p95=4684.10 max=4747.04
llm_time_per_token_ms: avg=15.29 p50=15.17 p95=19.59 max=20.37
llm_batch_wall_ms_total: avg=3147.42 p50=3123.76 p95=3994.52 max=4375.56
llm_decode_wall_ms_total: avg=3078.80 p50=3022.17 p95=3906.95 max=4317.64
tokens_per_batch_p50: avg=1.23 p50=1.00 p95=4.00 max=5.00
bicodec_decode_calls: avg=3.77 p50=4.00 p95=5.00 max=5.00
bicodec_decode_wall_ms_total: avg=2260.69 p50=2273.60 p95=3752.65 max=4027.10
bicodec_decode_interval_ms_p50: avg=665.58 p50=670.40 p95=1385.51 max=1447.70
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
admission_wait_ms: avg=66.23 p50=0.00 p95=293.75 max=355.34
admission_inflight_on_grant: avg=73.33 p50=90.00 p95=96.00 max=96.00
admission_waiters_on_grant: avg=13.33 p50=2.00 p95=31.00 max=32.00
admission_queue_depth_on_entry: avg=6.42 p50=0.00 p95=31.00 max=32.00
batch_avg_size: avg=7.30 p50=7.24 p95=7.60 max=7.67
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_workers_live: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_workers_target: avg=1.67 p50=2.00 p95=2.00 max=2.00
batch_queue_wait_ms_avg: avg=584.86 p50=592.71 p95=596.69 max=598.73
batch_compute_ms_avg: avg=118.01 p50=117.37 p95=121.36 max=121.92
batch_queue_depth_avg: avg=47.73 p50=47.84 p95=48.28 max=48.36
latency_over_timeline_ms: avg=123.83 p50=80.30 p95=359.66 max=422.13
timeline_share: llm=99.5% bicodec=65.2% other=0.0%
gpu: util_avg=91% util_max=100% mem_avg=51372MB mem_max=51511MB

results saved: /tmp/stream_admission_cap96.json
ERROR 02-19 04:18:55 [core_client.py:605] Engine core proc EngineCore_DP0 died unexpectedly, shutting down client.


**Evaluating throughput vs failure trade-off**


The strict cap showed the expected SLO behavior (much lower tail for admitted streams, but heavy overload rejection at `c=128`). I’m running a softer cap profile now (`max_inflight=112`, larger queue/wait budget) to find a production balance point.


python3 scripts/stress_test_streaming_runtime.py --levels 96,128 --text short --max-tokens 1024 --seed-base 2026 --requests-multiplier 6 --min-requests 256 --stream-decode-interval 48 --stream-window-size 128 --stream-min-semantic-first 10 --stream-crossfade-ms 50 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --bicodec-batch-workers 2 --bicodec-batch-scale-pending 56 --bicodec-batch-scale-mode dynamic --stream-admission-max-inflight 112 --stream-admission-max-queue 48 --stream-admission-max-wait-ms 400 --stream-admission-poll-ms 2 --output /tmp/stream_admission_cap112.json


Output:
Chunk ID: f9d761
Wall time: 1.0003 seconds
Process running with session ID 4222
Original token count: 0
Output:


Summary: auto


Output:
Chunk ID: fd0c7a
Wall time: 5.0014 seconds
Process running with session ID 4222
Original token count: 1362
Output:
[2026-02-19 04:19:08] INFO tts_runtime.py:408: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.25
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-19 04:19:09 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-19 04:19:09 [model.py:1561] Using max model len 4096
INFO 02-19 04:19:09 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-19 04:19:09 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:10 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:12 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:51051 backend=nccl
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:13 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A


Output:
Chunk ID: cd564d
Wall time: 5.0014 seconds
Process running with session ID 4222
Original token count: 349
Output:
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:13 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:14 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=581568) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=581568) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=581568)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=581568)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.27it/s]
(EngineCore_DP0 pid=581568)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.27it/s]
(EngineCore_DP0 pid=581568)
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:15 [default_loader.py:291] Loading weights took 0.86 seconds
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:16 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.404270 seconds


Soft-cap run is progressing; once this completes I’ll compare three profiles side-by-side (no cap, strict cap, soft cap) and pick the best production operating point for latency/SLO.


Output:
Chunk ID: a767cf
Wall time: 5.0007 seconds
Process running with session ID 4222
Original token count: 858
Output:
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:20 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:20 [backends.py:872] Dynamo bytecode transform time: 4.17 s
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:23 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.668 s
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:23 [monitor.py:34] torch.compile takes 4.84 s in total
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:24 [gpu_worker.py:356] Available KV cache memory: 18.06 GiB
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:24 [kv_cache_utils.py:1307] GPU KV cache size: 1,577,712 tokens
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:24 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 385.18x
(EngineCore_DP0 pid=581568)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 32.63it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 35.12it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 36.20it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 36.59it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 36.70it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 36.66it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 36.67it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 34.79it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:01<00:00, 34.88it/s]
(EngineCore_DP0 pid=581568)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 28.96it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 33.25it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 34.87it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 35.25it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.70it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 34.88it/s]
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:26 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=581568) INFO 02-19 04:19:26 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.88 seconds
[2026-02-19 04:19:28] INFO tts_runtime.py:417: Initializing prompt builder...
[2026-02-19 04:19:28] INFO tts_runtime.py:424: Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


**Continuing polling loop**


Output:
Chunk ID: 08beb8
Wall time: 5.0012 seconds
Process running with session ID 4222
Original token count: 157
Output:
[2026-02-19 04:19:36] INFO tts_runtime.py:434: Initializing TTS pipeline...
[2026-02-19 04:19:36] INFO tts_runtime.py:445: Initializing streaming pipeline...
[2026-02-19 04:19:36] INFO long_text_processor.py:112: LongTextProcessor initialized
[2026-02-19 04:19:36] INFO tts_runtime.py:504: ✅ TTS runtime initialized in 32955ms
[2026-02-19 04:19:36] INFO tts_runtime.py:505:    Model version: LLM
[2026-02-19 04:19:36] INFO tts_runtime.py:507:    Streaming admission: inflight<=112 queue<=48 wait<=400.0ms poll=2.0ms
[2026-02-19 04:19:36] INFO tts_runtime.py:526: Speaker globals cache startup precompute skipped (disabled)


**Benchmarking execution with polling**


Output:
Chunk ID: 6679d8
Wall time: 30.0019 seconds
Process running with session ID 4222
Original token count: 0
Output: