Output:
Chunk ID: b79a6b
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 1396
Output:

    Supports streaming with sliding window approach (like SNAC).
    """

    def __init__(
        self,
        device: str = "cuda",
        model_path: str = BICODEC_TOKENIZER_PATH,
        enable_batching: bool = False,  # For compatibility with streaming pipeline
    ):
        """
        Initialize BiCodec decoder.

        Args:
            device: Device for BiCodec model (cuda/cpu)
            model_path: Path to BiCodec model checkpoint
            enable_batching: Enable async batching for streaming (not used yet)
        """
        self.device = torch.device(device if torch.cuda.is_available() else "cpu")
        self.model_path = model_path
        requested_batching = bool(enable_batching)
        if _env_flag("VEENA3_BICODEC_BATCHING_FORCE_ON", False):
            requested_batching = True
        if _env_flag("VEENA3_BICODEC_BATCHING_FORCE_OFF", False):
            requested_batching = False
        self.enable_batching = requested_batching and self.device.type == "cuda"
        self.batch_max_size = _env_int("VEENA3_BICODEC_BATCH_MAX", 16, minimum=1)
        self.batch_timeout_s = _env_float("VEENA3_BICODEC_BATCH_TIMEOUT_MS", 6.0, minimum=0.1) / 1000.0

        # Async micro-batcher state (per event loop).
        self._batch_map_lock = threading.Lock()
        self._batch_queues: Dict[int, asyncio.Queue] = {}
        self._batch_workers: Dict[int, asyncio.Task] = {}
        self._batch_total_batches = 0
        self._batch_total_requests = 0
        self._batch_max_seen = 0
        self._batch_queue_wait_ms_total = 0.0
        self._batch_queue_wait_ms_min: Optional[float] = None
        self._batch_queue_wait_ms_max = 0.0
        self._batch_compute_ms_total = 0.0
        self._batch_compute_ms_min: Optional[float] = None
        self._batch_compute_ms_max = 0.0
        self._batch_queue_depth_total = 0
        self._batch_queue_depth_max = 0
        self._batch_inflight_requests = 0

        print(f"🎵 Loading BiCodec audio tokenizer from {model_path}...")
        print(f"   Device: {self.device}")

        # Initialize BiCodec tokenizer
        self.audio_tokenizer = BiCodecTokenizer(model_path, device=str(self.device))

        # OPTIMIZATION: Ensure model is on device once at init, not per-decode call
        self.audio_tokenizer.device = self.device
        self.audio_tokenizer.model.to(self.device)

        # NOTE: torch.compile disabled for BiCodec -- dynamic semantic sequence lengths
        # cause repeated recompilation (~120s each), worse than eager mode.
        # torch.compile works best with static shapes. If needed, pre-warm with
        # representative lengths or use mode="default" instead of "reduce-overhead".
        # The multi-engine approach (Tier 3) provides more impactful gains.

        print(f"✅ BiCodec decoder initialized (sample rate: {AUDIO_SAMPLE_RATE}Hz)")
        if self.enable_batching:
            print(
                "   ℹ  Micro-batching enabled "
                f"(max_batch={self.batch_max_size}, timeout_ms={self.batch_timeout_s * 1000:.1f})"
            )

    def decode(
        self,
        semantic_ids: List[int],
        global_ids: List[int],
    ) -> Optional[np.ndarray]:
        """
        Decode BiCodec tokens to audio waveform.

        CRITICAL: BiCodec expects EXACTLY 32 global tokens!
        The decoder pools them via speaker_encoder → d_vector, then broadcasts
        via d_vector.unsqueeze(-1) across the time dimension.

        Args:
            semantic_ids: List of semantic token IDs (variable length)
            global_ids: List of global token IDs (MUST be exactly 32!)

        Returns:
            Audio waveform as numpy array (float32, 16kHz mono)
            Shape: (samples,)
            Returns None if decode fails
        """
        if not semantic_ids or not global_ids:
            print(f"⚠  Empty token lists: semantic={len(semantic_ids)}, global={len(global_ids)}")
            return None

        # Validate global token count
        EXPECTED_GLOBAL_TOKENS = 32
        if len(global_ids) != EXPECTED_GLOBAL_TOKENS:
            print(f"❌ BiCodec requires EXACTLY {EXPECTED_GLOBAL_TOKENS} global tokens, got {len(global_ids)}")
            # Try to fix it
            if len(global_ids) > EXPECTED_GLOBAL_TOKENS:
                global_ids = global_ids[:EXPECTED_GLOBAL_TOKENS]
                print(f"   └─ Truncated to {EXPECTED_GLOBAL_TOKENS}")
            else:
                global_ids = global_ids + [0] * (EXPECTED_GLOBAL_TOKENS - len(global_ids))
                print(f"   └─ Padded to {EXPECTED_GLOBAL_TOKENS}")

        try:
            # Convert to tensors following official Spark-TTS implementation
            # global_token_ids shape: (1, 32) - ALWAYS 32!
            # pred_semantic_ids shape: (1, N) - variable length
            pred_semantic = torch.tensor(semantic_ids).long().unsqueeze(0).to(self.device)
            pred_global = torch.tensor(global_ids).long().unsqueeze(0).to(self.device)

            # Device migration done once in __init__, not per-call
            # Decode through BiCodec
            # BiCodecTokenizer.detokenize() expects:
            #   - global_tokens: (batch, global_dim) -> will unsqueeze(1) internally
            #   - semantic_tokens: (batch, latent_dim)
            with torch.inference_mode():
                wav_np = self.audio_tokenizer.detokenize(
                    pred_global,  # (1, seq) - correct shape!


Output:
Chunk ID: 3e5ae0
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 1691
Output:
                if isinstance(audio, torch.Tensor):
                    audio = audio.detach().cpu().numpy()
                audio_np = np.asarray(audio).reshape(-1)
                expected_samples = sem_len * 320
                if expected_samples > 0 and len(audio_np) > expected_samples:
                    audio_np = audio_np[:expected_samples]
                audio_np = np.clip(audio_np, -1.0, 1.0)
                audio_int16 = (audio_np * 32767).astype(np.int16)
                results.append(audio_int16.tobytes())
            return results
        except Exception:
            logger.exception("BiCodec batched decode failed; falling back to sequential decode")
            return [
                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
                for sem, glob, use_sw, trim in batch_items
            ]

    def _ensure_batch_worker(self, loop: asyncio.AbstractEventLoop) -> asyncio.Queue:
        key = id(loop)
        with self._batch_map_lock:
            queue = self._batch_queues.get(key)
            worker = self._batch_workers.get(key)
            if queue is not None and worker is not None and not worker.done():
                return queue

            queue = asyncio.Queue(maxsize=max(self.batch_max_size * 8, 256))
            worker = loop.create_task(self._batch_worker(loop, queue), name=f"bicodec-batch-worker-{key}")
            self._batch_queues[key] = queue
            self._batch_workers[key] = worker
            return queue

    async def _batch_worker(self, loop: asyncio.AbstractEventLoop, queue: asyncio.Queue) -> None:
        key = id(loop)
        try:
            while True:
                first: _BatchDecodeRequest = await queue.get()
                batch: List[_BatchDecodeRequest] = [first]
                deadline = loop.time() + self.batch_timeout_s

                while len(batch) < self.batch_max_size:
                    remaining = deadline - loop.time()
                    if remaining <= 0:
                        break
                    try:
                        req: _BatchDecodeRequest = await asyncio.wait_for(queue.get(), timeout=remaining)
                    except asyncio.TimeoutError:
                        break
                    batch.append(req)

                batch_inputs = [
                    (req.semantic_ids, req.global_ids, req.use_sliding_window, req.trim_warmup)
                    for req in batch
                ]
                batch_start = time.perf_counter()
                queue_wait_ms = [
                    max(0.0, (batch_start - req.submitted_at) * 1000.0)
                    for req in batch
                ]
                queue_depth_now = queue.qsize()
                self._batch_queue_depth_total += queue_depth_now
                if queue_depth_now > self._batch_queue_depth_max:
                    self._batch_queue_depth_max = queue_depth_now
                for wait_ms in queue_wait_ms:
                    self._batch_queue_wait_ms_total += wait_ms
                    if self._batch_queue_wait_ms_min is None or wait_ms < self._batch_queue_wait_ms_min:
                        self._batch_queue_wait_ms_min = wait_ms
                    if wait_ms > self._batch_queue_wait_ms_max:
                        self._batch_queue_wait_ms_max = wait_ms
                with self._batch_map_lock:
                    self._batch_inflight_requests += len(batch)

                t0 = time.perf_counter()
                try:
                    results = await loop.run_in_executor(None, self._decode_streaming_batch_sync, batch_inputs)
                    if len(results) != len(batch):
                        raise RuntimeError("batch decode result size mismatch")
                    for req, result in zip(batch, results):
                        if not req.future.done():
                            req.future.set_result(result)
                except Exception as exc:
                    for req in batch:
                        if not req.future.done():
                            req.future.set_exception(exc)
                finally:
                    compute_ms = (time.perf_counter() - t0) * 1000.0
                    self._batch_compute_ms_total += compute_ms
                    if self._batch_compute_ms_min is None or compute_ms < self._batch_compute_ms_min:
                        self._batch_compute_ms_min = compute_ms
                    if compute_ms > self._batch_compute_ms_max:
                        self._batch_compute_ms_max = compute_ms
                    self._batch_total_batches += 1
                    self._batch_total_requests += len(batch)
                    if len(batch) > self._batch_max_seen:
                        self._batch_max_seen = len(batch)
                    with self._batch_map_lock:
                        self._batch_inflight_requests = max(0, self._batch_inflight_requests - len(batch))
        finally:
            with self._batch_map_lock:
                self._batch_queues.pop(key, None)
                self._batch_workers.pop(key, None)

    def get_pending_requests(self) -> int:
        if not self.enable_batching:
            return 0
        with self._batch_map_lock:
            pending = sum(queue.qsize() for queue in self._batch_queues.values())
            return int(max(0, pending + self._batch_inflight_requests))

    def get_batching_stats(self) -> Dict[str, float]:
        batches = max(1, self._batch_total_batches)
        reqs = max(1, self._batch_total_requests)
        return {
            "enabled": 1.0 if self.enable_batching else 0.0,
            "batch_total_batches": float(self._batch_total_batches),
            "batch_total_requests": float(self._batch_total_requests),
            "batch_avg_size": float(self._batch_total_requests / batches),
            "batch_max_seen": float(self._batch_max_seen),
            "batch_queue_wait_ms_avg": float(self._batch_queue_wait_ms_total / reqs),
            "batch_queue_wait_ms_min": float(self._batch_queue_wait_ms_min or 0.0),
            "batch_queue_wait_ms_max": float(self._batch_queue_wait_ms_max),
            "batch_compute_ms_avg": float(self._batch_compute_ms_total / batches),
            "batch_compute_ms_min": float(self._batch_compute_ms_min or 0.0),
            "batch_compute_ms_max": float(self._batch_compute_ms_max),
            "batch_queue_depth_avg": float(self._batch_queue_depth_total / batches),
            "batch_queue_depth_max": float(self._batch_queue_depth_max),
            "batch_pending_now": float(self.get_pending_requests()),
        }

    def validate_tokens(self, semantic_ids: List[int], global_ids: List[int]) -> bool:
        """
        Validate BiCodec tokens before decoding.


Output:
Chunk ID: 5cca52
Wall time: 0.0505 seconds
Process exited with code 0
Original token count: 987
Output:
    parser.add_argument("--gpu-memory", type=float, default=0.25)
    parser.add_argument("--stream-output-kind", choices=["delta", "cumulative"], default="delta")
    parser.add_argument("--num-engines", type=int, default=1, help="Number of vLLM engines")
    parser.add_argument("--max-num-batched-tokens", type=int, default=None)
    parser.add_argument("--max-num-seqs", type=int, default=None)
    parser.add_argument("--enable-chunked-prefill", action="store_true", default=None)
    parser.add_argument("--disable-chunked-prefill", action="store_true", default=False)
    parser.add_argument("--disable-prefix-caching", action="store_true", default=False)
    parser.add_argument("--disable-engine-stats-logs", action="store_true", default=False)
    parser.add_argument("--enforce-eager", action="store_true", default=False)
    parser.add_argument("--stream-decode-interval", type=int, default=48)
    parser.add_argument("--stream-window-size", type=int, default=128)
    parser.add_argument("--stream-min-semantic-first", type=int, default=10)
    parser.add_argument("--stream-crossfade-ms", type=int, default=50)
    parser.add_argument("--adaptive-decode", action="store_true", default=False)
    parser.add_argument("--adaptive-decode-first-interval", type=int, default=48)
    parser.add_argument("--adaptive-decode-busy-interval", type=int, default=64)
    parser.add_argument("--adaptive-decode-busy-pending", type=int, default=32)
    parser.add_argument("--disable-windowed-decode", action="store_true", default=False)
    parser.add_argument("--disable-bicodec-batching", action="store_true", default=False)
    parser.add_argument("--bicodec-batch-max", type=int, default=16)
    parser.add_argument("--bicodec-batch-timeout-ms", type=float, default=6.0)
    parser.add_argument("--model-path", type=str, default="")
    parser.add_argument("--output", default="stress_streaming_runtime.json")
    args = parser.parse_args()

    from veena3modal.local_server import DEFAULT_LOCAL_MODEL_DIR

    model_path = args.model_path or DEFAULT_LOCAL_MODEL_DIR

    os.environ["VEENA3_STREAM_OUTPUT_KIND"] = args.stream_output_kind
    os.environ["VEENA3_STREAM_DECODE_INTERVAL"] = str(args.stream_decode_interval)
    os.environ["VEENA3_STREAM_WINDOW_SIZE"] = str(args.stream_window_size)
    os.environ["VEENA3_STREAM_MIN_SEMANTIC_FIRST"] = str(args.stream_min_semantic_first)
    os.environ["VEENA3_STREAM_CROSSFADE_MS"] = str(args.stream_crossfade_ms)
    os.environ["VEENA3_STREAM_ADAPTIVE_DECODE"] = "1" if args.adaptive_decode else "0"
    os.environ["VEENA3_STREAM_DECODE_INTERVAL_FIRST"] = str(args.adaptive_decode_first_interval)
    os.environ["VEENA3_STREAM_DECODE_INTERVAL_BUSY"] = str(args.adaptive_decode_busy_interval)
    os.environ["VEENA3_STREAM_DECODE_BUSY_PENDING"] = str(args.adaptive_decode_busy_pending)
    os.environ["VEENA3_STREAM_WINDOWED_DECODE"] = "0" if args.disable_windowed_decode else "1"
    os.environ["VEENA3_BICODEC_BATCHING"] = "0" if args.disable_bicodec_batching else "1"
    os.environ["VEENA3_BICODEC_BATCH_MAX"] = str(args.bicodec_batch_max)
    os.environ["VEENA3_BICODEC_BATCH_TIMEOUT_MS"] = str(args.bicodec_batch_timeout_ms)

    enable_chunked_prefill = args.enable_chunked_prefill
    if args.disable_chunked_prefill:
        enable_chunked_prefill = False

    init_runtime(
        model_path=model_path,
        gpu_mem=args.gpu_memory,
        stream_output_kind=args.stream_output_kind,
        num_engines=args.num_engines,
        max_num_batched_tokens=args.max_num_batched_tokens,
        max_num_seqs=args.max_num_seqs,
        enable_chunked_prefill=enable_chunked_prefill,
        enable_prefix_caching=(False if args.disable_prefix_caching else None),
        disable_log_stats=(True if args.disable_engine_stats_logs else None),
        enforce_eager=(True if args.enforce_eager else None),
    )
    asyncio.run(main_async(args))

if __name__ == "__main__":
    main()


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Detailing multi-worker batching plan**


I’m now editing `bicodec_decoder.py` to turn the single worker into a configurable worker pool on the same shared queue, with live-worker stats so we can validate whether parallel decode workers reduce queue wait under high concurrency.


Input:
*** Begin Patch
*** Update File: veena3modal/core/bicodec_decoder.py
@@
-        self.batch_max_size = _env_int("VEENA3_BICODEC_BATCH_MAX", 16, minimum=1)
+        self.batch_max_size = _env_int("VEENA3_BICODEC_BATCH_MAX", 16, minimum=1)
         self.batch_timeout_s = _env_float("VEENA3_BICODEC_BATCH_TIMEOUT_MS", 6.0, minimum=0.1) / 1000.0
+        self.batch_worker_count = _env_int("VEENA3_BICODEC_BATCH_WORKERS", 1, minimum=1)

         # Async micro-batcher state (per event loop).
         self._batch_map_lock = threading.Lock()
         self._batch_queues: Dict[int, asyncio.Queue] = {}
-        self._batch_workers: Dict[int, asyncio.Task] = {}
+        self._batch_workers: Dict[int, List[asyncio.Task]] = {}
@@
                 "   ℹ  Micro-batching enabled "
-                f"(max_batch={self.batch_max_size}, timeout_ms={self.batch_timeout_s * 1000:.1f})"
+                f"(workers={self.batch_worker_count}, max_batch={self.batch_max_size}, "
+                f"timeout_ms={self.batch_timeout_s * 1000:.1f})"
             )
@@
     def _ensure_batch_worker(self, loop: asyncio.AbstractEventLoop) -> asyncio.Queue:
         key = id(loop)
         with self._batch_map_lock:
             queue = self._batch_queues.get(key)
-            worker = self._batch_workers.get(key)
-            if queue is not None and worker is not None and not worker.done():
-                return queue
-
-            queue = asyncio.Queue(maxsize=max(self.batch_max_size * 8, 256))
-            worker = loop.create_task(self._batch_worker(loop, queue), name=f"bicodec-batch-worker-{key}")
-            self._batch_queues[key] = queue
-            self._batch_workers[key] = worker
+            workers = self._batch_workers.get(key, [])
+            live_workers = [worker for worker in workers if not worker.done()]
+
+            if queue is None:
+                queue = asyncio.Queue(maxsize=max(self.batch_max_size * 8, 256))
+                self._batch_queues[key] = queue
+
+            while len(live_workers) < self.batch_worker_count:
+                worker_idx = len(live_workers)
+                worker = loop.create_task(
+                    self._batch_worker(loop, queue, worker_idx),
+                    name=f"bicodec-batch-worker-{key}-{worker_idx}",
+                )
+                live_workers.append(worker)
+
+            self._batch_workers[key] = live_workers
             return queue

-    async def _batch_worker(self, loop: asyncio.AbstractEventLoop, queue: asyncio.Queue) -> None:
-        key = id(loop)
-        try:
-            while True:
-                first: _BatchDecodeRequest = await queue.get()
-                batch: List[_BatchDecodeRequest] = [first]
-                deadline = loop.time() + self.batch_timeout_s
-
-                while len(batch) < self.batch_max_size:
-                    remaining = deadline - loop.time()
-                    if remaining <= 0:
-                        break
-                    try:
-                        req: _BatchDecodeRequest = await asyncio.wait_for(queue.get(), timeout=remaining)
-                    except asyncio.TimeoutError:
-                        break
-                    batch.append(req)
-
-                batch_inputs = [
-                    (req.semantic_ids, req.global_ids, req.use_sliding_window, req.trim_warmup)
-                    for req in batch
-                ]
-                batch_start = time.perf_counter()
-                queue_wait_ms = [
-                    max(0.0, (batch_start - req.submitted_at) * 1000.0)
-                    for req in batch
-                ]
-                queue_depth_now = queue.qsize()
-                self._batch_queue_depth_total += queue_depth_now
-                if queue_depth_now > self._batch_queue_depth_max:
-                    self._batch_queue_depth_max = queue_depth_now
-                for wait_ms in queue_wait_ms:
-                    self._batch_queue_wait_ms_total += wait_ms
-                    if self._batch_queue_wait_ms_min is None or wait_ms < self._batch_queue_wait_ms_min:
-                        self._batch_queue_wait_ms_min = wait_ms
-                    if wait_ms > self._batch_queue_wait_ms_max:
-                        self._batch_queue_wait_ms_max = wait_ms
-                with self._batch_map_lock:
-                    self._batch_inflight_requests += len(batch)
-
-                t0 = time.perf_counter()
-                try:
-                    results = await loop.run_in_executor(None, self._decode_streaming_batch_sync, batch_inputs)
-                    if len(results) != len(batch):
-                        raise RuntimeError("batch decode result size mismatch")
-                    for req, result in zip(batch, results):
-                        if not req.future.done():
-                            req.future.set_result(result)
-                except Exception as exc:
-                    for req in batch:
-                        if not req.future.done():
-                            req.future.set_exception(exc)
-                finally:
-                    compute_ms = (time.perf_counter() - t0) * 1000.0
-                    self._batch_compute_ms_total += compute_ms
-                    if self._batch_compute_ms_min is None or compute_ms < self._batch_compute_ms_min:
-                        self._batch_compute_ms_min = compute_ms
-                    if compute_ms > self._batch_compute_ms_max:
-                        self._batch_compute_ms_max = compute_ms
-                    self._batch_total_batches += 1
-                    self._batch_total_requests += len(batch)
-                    if len(batch) > self._batch_max_seen:
-                        self._batch_max_seen = len(batch)
-                    with self._batch_map_lock:
-                        self._batch_inflight_requests = max(0, self._batch_inflight_requests - len(batch))
-        finally:
-            with self._batch_map_lock:
-                self._batch_queues.pop(key, None)
-                self._batch_workers.pop(key, None)
+    async def _batch_worker(
+        self,
+        loop: asyncio.AbstractEventLoop,
+        queue: asyncio.Queue,
+        worker_idx: int,
+    ) -> None:
+        while True:
+            first: _BatchDecodeRequest = await queue.get()
+            batch: List[_BatchDecodeRequest] = [first]
+            deadline = loop.time() + self.batch_timeout_s
+
+            while len(batch) < self.batch_max_size:
+                remaining = deadline - loop.time()
+                if remaining <= 0:
+                    break
+                try:
+                    req: _BatchDecodeRequest = await asyncio.wait_for(queue.get(), timeout=remaining)
+                except asyncio.TimeoutError:
+                    break
+                batch.append(req)
+
+            batch_inputs = [
+                (req.semantic_ids, req.global_ids, req.use_sliding_window, req.trim_warmup)
+                for req in batch
+            ]
+            batch_start = time.perf_counter()
+            queue_wait_ms = [
+                max(0.0, (batch_start - req.submitted_at) * 1000.0)
+                for req in batch
+            ]
+            queue_depth_now = queue.qsize()
+            self._batch_queue_depth_total += queue_depth_now
+            if queue_depth_now > self._batch_queue_depth_max:
+                self._batch_queue_depth_max = queue_depth_now
+            for wait_ms in queue_wait_ms:
+                self._batch_queue_wait_ms_total += wait_ms
+                if self._batch_queue_wait_ms_min is None or wait_ms < self._batch_queue_wait_ms_min:
+                    self._batch_queue_wait_ms_min = wait_ms
+                if wait_ms > self._batch_queue_wait_ms_max:
+                    self._batch_queue_wait_ms_max = wait_ms
+            with self._batch_map_lock:
+                self._batch_inflight_requests += len(batch)
+
+            t0 = time.perf_counter()
+            try:
+                results = await loop.run_in_executor(None, self._decode_streaming_batch_sync, batch_inputs)
+                if len(results) != len(batch):
+                    raise RuntimeError("batch decode result size mismatch")
+                for req, result in zip(batch, results):
+                    if not req.future.done():
+                        req.future.set_result(result)
+            except Exception as exc:
+                logger.exception("BiCodec batch worker %d failed", worker_idx)
+                for req in batch:
+                    if not req.future.done():
+                        req.future.set_exception(exc)
+            finally:
+                compute_ms = (time.perf_counter() - t0) * 1000.0
+                self._batch_compute_ms_total += compute_ms
+                if self._batch_compute_ms_min is None or compute_ms < self._batch_compute_ms_min:
+                    self._batch_compute_ms_min = compute_ms
+                if compute_ms > self._batch_compute_ms_max:
+                    self._batch_compute_ms_max = compute_ms
+                self._batch_total_batches += 1
+                self._batch_total_requests += len(batch)
+                if len(batch) > self._batch_max_seen:
+                    self._batch_max_seen = len(batch)
+                with self._batch_map_lock:
+                    self._batch_inflight_requests = max(0, self._batch_inflight_requests - len(batch))
@@
     def get_batching_stats(self) -> Dict[str, float]:
         batches = max(1, self._batch_total_batches)
         reqs = max(1, self._batch_total_requests)
+        with self._batch_map_lock:
+            workers_live = sum(
+                1
+                for workers in self._batch_workers.values()
+                for worker in workers
+                if not worker.done()
+            )
         return {
             "enabled": 1.0 if self.enable_batching else 0.0,
             "batch_total_batches": float(self._batch_total_batches),
             "batch_total_requests": float(self._batch_total_requests),
             "batch_avg_size": float(self._batch_total_requests / batches),
             "batch_max_seen": float(self._batch_max_seen),
+            "batch_workers_configured": float(self.batch_worker_count),
+            "batch_workers_live": float(workers_live),
             "batch_queue_wait_ms_avg": float(self._batch_queue_wait_ms_total / reqs),
             "batch_queue_wait_ms_min": float(self._batch_queue_wait_ms_min or 0.0),
             "batch_queue_wait_ms_max": float(self._batch_queue_wait_ms_max),
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/bicodec_decoder.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Summary: auto


**Adjusting cancellation exception handling**


Input:
*** Begin Patch
*** Update File: veena3modal/core/bicodec_decoder.py
@@
-            except Exception as exc:
+            except asyncio.CancelledError:
+                raise
+            except Exception as exc:
                 logger.exception("BiCodec batch worker %d failed", worker_idx)
                 for req in batch:
                     if not req.future.done():
                         req.future.set_exception(exc)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/bicodec_decoder.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Planning batch worker reuse fix**


I’ve added worker-pool support in the decoder; next I’m wiring `--bicodec-batch-workers` into the stress harness so we can run clean 1-worker vs 2-worker benchmarks with identical seeds.


Input:
*** Begin Patch
*** Update File: scripts/stress_test_streaming_runtime.py
@@
         "batch_avg_size",
         "batch_max_seen",
+        "batch_workers_live",
         "batch_queue_wait_ms_avg",
         "batch_compute_ms_avg",
         "batch_queue_depth_avg",
     ]
@@
             "disable_bicodec_batching": args.disable_bicodec_batching,
             "bicodec_batch_max": args.bicodec_batch_max,
             "bicodec_batch_timeout_ms": args.bicodec_batch_timeout_ms,
+            "bicodec_batch_workers": args.bicodec_batch_workers,
             "timestamp": int(time.time()),
         },
@@
                     "decode_pending_requests_p50": lvl.timing_stats("decode_pending_requests_p50"),
                     "batch_avg_size": lvl.timing_stats("batch_avg_size"),
                     "batch_max_seen": lvl.timing_stats("batch_max_seen"),
+                    "batch_workers_live": lvl.timing_stats("batch_workers_live"),
                     "batch_queue_wait_ms_avg": lvl.timing_stats("batch_queue_wait_ms_avg"),
                     "batch_compute_ms_avg": lvl.timing_stats("batch_compute_ms_avg"),
                     "batch_queue_depth_avg": lvl.timing_stats("batch_queue_depth_avg"),
@@
     parser.add_argument("--disable-bicodec-batching", action="store_true", default=False)
     parser.add_argument("--bicodec-batch-max", type=int, default=16)
     parser.add_argument("--bicodec-batch-timeout-ms", type=float, default=6.0)
+    parser.add_argument("--bicodec-batch-workers", type=int, default=1)
     parser.add_argument("--model-path", type=str, default="")
@@
     os.environ["VEENA3_BICODEC_BATCHING"] = "0" if args.disable_bicodec_batching else "1"
     os.environ["VEENA3_BICODEC_BATCH_MAX"] = str(args.bicodec_batch_max)
     os.environ["VEENA3_BICODEC_BATCH_TIMEOUT_MS"] = str(args.bicodec_batch_timeout_ms)
+    os.environ["VEENA3_BICODEC_BATCH_WORKERS"] = str(args.bicodec_batch_workers)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Identifying validation and environment loading needs**


python3 -m py_compile veena3modal/core/bicodec_decoder.py scripts/stress_test_streaming_runtime.py veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: f5a6ae
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 0
Output:


**Planning benchmark runs**


Code compiles. I’m running seeded A/B now with identical settings except `bicodec_batch_workers=1` vs `2` at `c=96,128`, and I’ll report the throughput/latency/queue-wait deltas directly.


set -euo pipefail
for w in 1 2; do
  out="/tmp/stream_workers${w}_seeded.json"
  log="/tmp/stream_workers${w}_seeded.log"
  echo "=== RUN workers=${w} ==="
  python3 scripts/stress_test_streaming_runtime.py \
    --levels 96,128 \
    --text short \
    --max-tokens 1024 \
    --seed-base 2026 \
    --timeout 300 \
    --requests-multiplier 1 \
    --min-requests 8 \
    --gpu-memory 0.25 \
    --stream-output-kind delta \
    --stream-decode-interval 48 \
    --stream-window-size 128 \
    --bicodec-batch-max 16 \
    --bicodec-batch-timeout-ms 6 \
    --bicodec-batch-workers "$w" \
    --output "$out" >"$log" 2>&1
  python3 - <<PY
import json
p='$out'
d=json.load(open(p))
print('file',p)
for r in d['levels']:
    t=r['timing']
    print(f"c={r['concurrency']} thr={r['throughput_rps']:.2f} lat={r['latency_ms']['avg']:.0f} ttfb={r['ttfb_ms']['avg']:.0f} llm_tok_ms={t['llm_time_per_token_ms']['avg']:.2f} bicodec={t['bicodec_decode_wall_ms_total']['avg']:.0f} batch_avg={t['batch_avg_size']['avg']:.2f} workers_live={t.get('batch_workers_live',{}).get('avg',0):.1f} qwait={t['batch_queue_wait_ms_avg']['avg']:.0f} qdepth={t['batch_queue_depth_avg']['avg']:.1f}")
PY
done


Output:
Chunk ID: 6eef6b
Wall time: 1.0013 seconds
Process running with session ID 52488
Original token count: 6
Output:
=== RUN workers=1 ===


**Planning periodic polling**


Output:
Chunk ID: b50736
Wall time: 5.0002 seconds
Process running with session ID 52488
Original token count: 0
Output:


Output:
Chunk ID: d084ec
Wall time: 5.0003 seconds
Process running with session ID 52488
Original token count: 0
Output:


The first A/B run (`workers=1`) is still in startup/warmup. As soon as it finishes I’ll immediately run and compare the `workers=2` result with the same seeded load profile.


Output:
Chunk ID: 763fe5
Wall time: 5.0007 seconds
Process running with session ID 52488
Original token count: 0
Output:


Output:
Chunk ID: 64b111
Wall time: 5.0008 seconds
Process running with session ID 52488
Original token count: 0
Output:


Output:
Chunk ID: 7b5599
Wall time: 5.0011 seconds
Process running with session ID 52488
Original token count: 75
Output:
file /tmp/stream_workers1_seeded.json
c=96 thr=23.05 lat=3627 ttfb=1155 llm_tok_ms=15.59 bicodec=2344 batch_avg=10.03 workers_live=1.0 qwait=478 qdepth=45.3
c=128 thr=22.79 lat=4619 ttfb=1520 llm_tok_ms=19.94 bicodec=3567 batch_avg=8.52 workers_live=1.0 qwait=694 qdepth=68.4
=== RUN workers=2 ===


**Continuing poll and commentary update**


Baseline for this A/B (`workers=1`) is in: `23.05 req/s` at c96 and `22.79 req/s` at c128, with high queue wait (`478ms` / `694ms`). I’m now on `workers=2` to check if that queue wait collapses.


Output:
Chunk ID: 5c0add
Wall time: 5.0017 seconds
Process running with session ID 52488
Original token count: 0
Output:


Output:
Chunk ID: df9f18
Wall time: 5.0011 seconds
Process running with session ID 52488
Original token count: 0
Output:


Output:
Chunk ID: 6a7078
Wall time: 5.0012 seconds
Process running with session ID 52488
Original token count: 0
Output:


`workers=2` run is still executing. After it completes I’ll produce a strict delta table (`w2 - w1`) for throughput, latency, TTFB, queue wait, and queue depth at both concurrencies.


Output:
Chunk ID: 303984
Wall time: 5.0007 seconds
Process running with session ID 52488
Original token count: 0
Output:


Output:
Chunk ID: 85a416
Wall time: 5.0010 seconds
Process running with session ID 52488
Original token count: 0
Output:


Output:
Chunk ID: 7c161a
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 69
Output:
file /tmp/stream_workers2_seeded.json
c=96 thr=22.25 lat=3763 ttfb=1151 llm_tok_ms=16.05 bicodec=2677 batch_avg=8.46 workers_live=2.0 qwait=502 qdepth=48.0
c=128 thr=23.12 lat=4565 ttfb=1383 llm_tok_ms=19.47 bicodec=2979 batch_avg=8.53 workers_live=2.0 qwait=576 qdepth=57.0


**Planning worker count tuning**


python3 - <<'PY'
import json
w1=json.load(open('/tmp/stream_workers1_seeded.json'))['levels']
w2=json.load(open('/tmp/stream_workers2_seeded.json'))['levels']
by1={r['concurrency']:r for r in w1}
by2={r['concurrency']:r for r in w2}
for c in sorted(by1):
    a=by1[c]; b=by2[c]
    ta=a['timing']; tb=b['timing']
    print(f"c={c}")
    print(f"  thr: {a['throughput_rps']:.2f} -> {b['throughput_rps']:.2f} ({(b['throughput_rps']-a['throughput_rps'])/a['throughput_rps']*100:+.1f}%)")
    print(f"  lat: {a['latency_ms']['avg']:.0f} -> {b['latency_ms']['avg']:.0f} ({(b['latency_ms']['avg']-a['latency_ms']['avg'])/a['latency_ms']['avg']*100:+.1f}%)")
    print(f"  ttfb: {a['ttfb_ms']['avg']:.0f} -> {b['ttfb_ms']['avg']:.0f} ({(b['ttfb_ms']['avg']-a['ttfb_ms']['avg'])/a['ttfb_ms']['avg']*100:+.1f}%)")
    print(f"  qwait: {ta['batch_queue_wait_ms_avg']['avg']:.0f} -> {tb['batch_queue_wait_ms_avg']['avg']:.0f} ({(tb['batch_queue_wait_ms_avg']['avg']-ta['batch_queue_wait_ms_avg']['avg'])/ta['batch_queue_wait_ms_avg']['avg']*100:+.1f}%)")
    print(f"  qdepth: {ta['batch_queue_depth_avg']['avg']:.1f} -> {tb['batch_queue_depth_avg']['avg']:.1f} ({(tb['batch_queue_depth_avg']['avg']-ta['batch_queue_depth_avg']['avg'])/ta['batch_queue_depth_avg']['avg']*100:+.1f}%)")
    print(f"  bicodec: {ta['bicodec_decode_wall_ms_total']['avg']:.0f} -> {tb['bicodec_decode_wall_ms_total']['avg']:.0f} ({(tb['bicodec_decode_wall_ms_total']['avg']-ta['bicodec_decode_wall_ms_total']['avg'])/ta['bicodec_decode_wall_ms_total']['avg']*100:+.1f}%)")
PY


Output:
Chunk ID: 241751
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 93
Output:
c=96
  thr: 23.05 -> 22.25 (-3.4%)
  lat: 3627 -> 3763 (+3.7%)
  ttfb: 1155 -> 1151 (-0.3%)
  qwait: 478 -> 502 (+5.0%)
  qdepth: 45.3 -> 48.0 (+6.0%)
  bicodec: 2344 -> 2677 (+14.2%)
c=128
  thr: 22.79 -> 23.12 (+1.4%)
  lat: 4619 -> 4565 (-1.2%)
  ttfb: 1520 -> 1383 (-9.0%)
  qwait: 694 -> 576 (-17.0%)
  qdepth: 68.4 -> 57.0 (-16.7%)
  bicodec: 3567 -> 2979 (-16.5%)


**Designing dynamic batch worker scaling**


`workers=2` helps only at very high load (`c=128`) and hurts at `c=96`. I’m implementing auto-scale workers (spawn second worker only when pending decode queue crosses a threshold) so we can keep `c=96` efficiency and still relieve `c=128` queue pressure.


Input:
*** Begin Patch
*** Update File: veena3modal/core/bicodec_decoder.py
@@
         self.batch_max_size = _env_int("VEENA3_BICODEC_BATCH_MAX", 16, minimum=1)
         self.batch_timeout_s = _env_float("VEENA3_BICODEC_BATCH_TIMEOUT_MS", 6.0, minimum=0.1) / 1000.0
         self.batch_worker_count = _env_int("VEENA3_BICODEC_BATCH_WORKERS", 1, minimum=1)
+        self.batch_worker_scale_pending = _env_int("VEENA3_BICODEC_BATCH_SCALE_PENDING", 0, minimum=0)
@@
         self._batch_queue_depth_total = 0
         self._batch_queue_depth_max = 0
         self._batch_inflight_requests = 0
+        self._batch_workers_target_last = 1
@@
                 "   ℹ  Micro-batching enabled "
-                f"(workers={self.batch_worker_count}, max_batch={self.batch_max_size}, "
-                f"timeout_ms={self.batch_timeout_s * 1000:.1f})"
+                f"(workers={self.batch_worker_count}, scale_pending={self.batch_worker_scale_pending}, "
+                f"max_batch={self.batch_max_size}, timeout_ms={self.batch_timeout_s * 1000:.1f})"
             )
@@
     def _ensure_batch_worker(self, loop: asyncio.AbstractEventLoop) -> asyncio.Queue:
         key = id(loop)
         with self._batch_map_lock:
             queue = self._batch_queues.get(key)
             workers = self._batch_workers.get(key, [])
             live_workers = [worker for worker in workers if not worker.done()]

             if queue is None:
                 queue = asyncio.Queue(maxsize=max(self.batch_max_size * 8, 256))
                 self._batch_queues[key] = queue

-            while len(live_workers) < self.batch_worker_count:
+            target_workers = self.batch_worker_count
+            if self.batch_worker_count > 1 and self.batch_worker_scale_pending > 0:
+                pending_now = queue.qsize() + self._batch_inflight_requests
+                if pending_now < self.batch_worker_scale_pending:
+                    target_workers = 1
+            self._batch_workers_target_last = target_workers
+
+            while len(live_workers) < target_workers:
                 worker_idx = len(live_workers)
                 worker = loop.create_task(
                     self._batch_worker(loop, queue, worker_idx),
                     name=f"bicodec-batch-worker-{key}-{worker_idx}",
                 )
                 live_workers.append(worker)

+            while len(live_workers) > target_workers:
+                worker = live_workers.pop()
+                worker.cancel()
+
             self._batch_workers[key] = live_workers
             return queue
@@
             "batch_workers_configured": float(self.batch_worker_count),
             "batch_workers_live": float(workers_live),
+            "batch_workers_target": float(self._batch_workers_target_last),
+            "batch_worker_scale_pending": float(self.batch_worker_scale_pending),
             "batch_queue_wait_ms_avg": float(self._batch_queue_wait_ms_total / reqs),
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/bicodec_decoder.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Summary: auto


**Updating environment scale parameters**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_streaming_runtime.py
@@
         "batch_avg_size",
         "batch_max_seen",
         "batch_workers_live",
+        "batch_workers_target",
         "batch_queue_wait_ms_avg",
         "batch_compute_ms_avg",
         "batch_queue_depth_avg",
     ]
@@
             "bicodec_batch_max": args.bicodec_batch_max,
             "bicodec_batch_timeout_ms": args.bicodec_batch_timeout_ms,
             "bicodec_batch_workers": args.bicodec_batch_workers,
+            "bicodec_batch_scale_pending": args.bicodec_batch_scale_pending,
             "timestamp": int(time.time()),
         },
@@
                     "batch_avg_size": lvl.timing_stats("batch_avg_size"),
                     "batch_max_seen": lvl.timing_stats("batch_max_seen"),
                     "batch_workers_live": lvl.timing_stats("batch_workers_live"),
+                    "batch_workers_target": lvl.timing_stats("batch_workers_target"),
                     "batch_queue_wait_ms_avg": lvl.timing_stats("batch_queue_wait_ms_avg"),
                     "batch_compute_ms_avg": lvl.timing_stats("batch_compute_ms_avg"),
                     "batch_queue_depth_avg": lvl.timing_stats("batch_queue_depth_avg"),
@@
     parser.add_argument("--disable-bicodec-batching", action="store_true", default=False)
     parser.add_argument("--bicodec-batch-max", type=int, default=16)
     parser.add_argument("--bicodec-batch-timeout-ms", type=float, default=6.0)
     parser.add_argument("--bicodec-batch-workers", type=int, default=1)
+    parser.add_argument("--bicodec-batch-scale-pending", type=int, default=0)
     parser.add_argument("--model-path", type=str, default="")
@@
     os.environ["VEENA3_BICODEC_BATCH_MAX"] = str(args.bicodec_batch_max)
     os.environ["VEENA3_BICODEC_BATCH_TIMEOUT_MS"] = str(args.bicodec_batch_timeout_ms)
     os.environ["VEENA3_BICODEC_BATCH_WORKERS"] = str(args.bicodec_batch_workers)
+    os.environ["VEENA3_BICODEC_BATCH_SCALE_PENDING"] = str(args.bicodec_batch_scale_pending)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Recompiling after changes**


python3 -m py_compile veena3modal/core/bicodec_decoder.py scripts/stress_test_streaming_runtime.py


Output:
Chunk ID: 83b619
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 0
Output:


**Benchmarking with increased workers and scale**


Auto-scale logic is in and compiled. I’m running `max_workers=2` with `scale_pending=56` (second worker only when decode backlog is high) to test if we keep `c=96` efficiency while improving `c=128`.


python3 scripts/stress_test_streaming_runtime.py --levels 96,128 --text short --max-tokens 1024 --seed-base 2026 --timeout 300 --requests-multiplier 1 --min-requests 8 --gpu-memory 0.25 --stream-output-kind delta --stream-decode-interval 48 --stream-window-size 128 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --bicodec-batch-workers 2 --bicodec-batch-scale-pending 56 --output /tmp/stream_workers2_scale56_seeded.json


Output:
Chunk ID: ab5fa3
Wall time: 1.0011 seconds
Process running with session ID 23321
Original token count: 0
Output:


**Planning polling and updates**


Output:
Chunk ID: 5dc8e3
Wall time: 5.0006 seconds
Process running with session ID 23321
Original token count: 1362
Output:
[2026-02-18 18:47:38] INFO tts_runtime.py:183: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.25
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-18 18:47:39 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-18 18:47:39 [model.py:1561] Using max model len 4096
INFO 02-18 18:47:39 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-18 18:47:39 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:40 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:42 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:46759 backend=nccl
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:42 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A


Output:
Chunk ID: 6f05ba
Wall time: 5.0010 seconds
Process running with session ID 23321
Original token count: 349
Output:
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:43 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:44 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=104183) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=104183) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=104183)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=104183)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.34s/it]
(EngineCore_DP0 pid=104183)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.34s/it]
(EngineCore_DP0 pid=104183)
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:45 [default_loader.py:291] Loading weights took 1.42 seconds
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:46 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.936484 seconds


Output:
Chunk ID: 4fc782
Wall time: 5.0008 seconds
Process running with session ID 23321
Original token count: 798
Output:
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:50 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:50 [backends.py:872] Dynamo bytecode transform time: 4.18 s
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:53 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.643 s
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:53 [monitor.py:34] torch.compile takes 4.82 s in total
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:54 [gpu_worker.py:356] Available KV cache memory: 18.06 GiB
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:54 [kv_cache_utils.py:1307] GPU KV cache size: 1,577,712 tokens
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:54 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 385.18x
(EngineCore_DP0 pid=104183)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 33.37it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 36.31it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 37.53it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 38.04it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 37.46it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 36.96it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 36.97it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 35.52it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:00<00:00, 35.62it/s]
(EngineCore_DP0 pid=104183)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 26.34it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 31.59it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 34.29it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 35.37it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 36.09it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 34.65it/s]
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:57 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=104183) INFO 02-18 18:47:57 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.68 seconds
[2026-02-18 18:47:58] INFO tts_runtime.py:192: Initializing prompt builder...
[2026-02-18 18:47:58] INFO tts_runtime.py:199: Initializing BiCodec decoder...


The auto-scale run is in the generation phase now. Once it exits I’ll compare it against `workers=1` and `workers=2 always-on` to see if thresholded scaling gives the best of both.


Output:
Chunk ID: 5efedd
Wall time: 5.0011 seconds
Process running with session ID 23321
Original token count: 186
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)
[2026-02-18 18:48:06] INFO tts_runtime.py:209: Initializing TTS pipeline...
[2026-02-18 18:48:06] INFO tts_runtime.py:220: Initializing streaming pipeline...
[2026-02-18 18:48:06] INFO long_text_processor.py:112: LongTextProcessor initialized
[2026-02-18 18:48:06] INFO tts_runtime.py:277: ✅ TTS runtime initialized in 33141ms
[2026-02-18 18:48:06] INFO tts_runtime.py:278:    Model version: LLM
[2026-02-18 18:48:06] INFO tts_runtime.py:289: Speaker globals cache startup precompute skipped (disabled)


Output:
Chunk ID: 07f378
Wall time: 5.0011 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: 03906b
Wall time: 5.0006 seconds
Process running with session ID 23321
Original token count: 0
Output:


Still running; no summary line yet. I’ll extract and rank all three profiles immediately after completion (`w1`, `w2`, `w2+scale56`).


Output:
Chunk ID: e99778
Wall time: 5.0006 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: def6e9
Wall time: 5.0015 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: 532de4
Wall time: 5.0009 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: b5cab9
Wall time: 5.0018 seconds
Process running with session ID 23321
Original token count: 0
Output:


This run is taking longer than expected at higher load, which is usually when decode queue behavior shifts. I’m continuing to wait for completion so we can evaluate the scaling trigger effect with full metrics.


Output:
Chunk ID: 4edb31
Wall time: 5.0013 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: 341fcd
Wall time: 5.0016 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: eae372
Wall time: 5.0012 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: 3a30d6
Wall time: 5.0007 seconds
Process running with session ID 23321
Original token count: 0
Output:


No output yet, likely still processing the `c=128` segment. I’ll keep it running a bit longer and then verify process status if it remains silent.


Output:
Chunk ID: bf9a0d
Wall time: 5.0014 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: b7d5c1
Wall time: 5.0008 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: cde463
Wall time: 5.0014 seconds
Process running with session ID 23321
Original token count: 0
Output:


**Checking process status**


Output:
Chunk ID: e6a3dd
Wall time: 5.0013 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: 8a76ae
Wall time: 5.0015 seconds
Process running with session ID 23321
Original token count: 0
Output:


**Checking process status concurrently**


I’m checking process health now since there’s been a long silent window, to confirm this isn’t stalled before we wait further.


ps -eo pid,etime,cmd | rg -n "stress_test_streaming_runtime.py --levels 96,128|PID"


ls -lh /tmp/stream_workers2_scale56_seeded.json /tmp/stream_workers2_scale56_seeded.log 2>/dev/null || true


tail -n 40 /tmp/stream_workers2_scale56_seeded.log 2>/dev/null || true


Output:
Chunk ID: befb22
Wall time: 0.0516 seconds
Process exited with code 0
Original token count: 171
Output:
1:    PID     ELAPSED CMD
357: 104000       02:54 python3 scripts/stress_test_streaming_runtime.py --levels 96,128 --text short --max-tokens 1024 --seed-base 2026 --timeout 300 --requests-multiplier 1 --min-requests 8 --gpu-memory 0.25 --stream-output-kind delta --stream-decode-interval 48 --stream-window-size 128 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --bicodec-batch-workers 2 --bicodec-batch-scale-pending 56 --output /tmp/stream_workers2_scale56_seeded.json
469: 107563       00:00 /bin/bash -c ps -eo pid,etime,cmd | rg -n "stress_test_streaming_runtime.py --levels 96,128|PID"
471: 107566       00:00 rg -n stress_test_streaming_runtime.py --levels 96,128|PID


Output:
Chunk ID: f4b161
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 0
Output:


Output:
Chunk ID: d42df4
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 0
Output:


Summary: auto


**Waiting on process completion**


Output:
Chunk ID: c72ad6
Wall time: 5.0014 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: a32548
Wall time: 5.0006 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: 53275d
Wall time: 5.0008 seconds
Process running with session ID 23321
Original token count: 0
Output:


The run is still active and likely waiting on slow-tail requests at `c=128` (timeout budget is 300s). I’ll let it finish once, then we can tighten test duration for faster iteration.


Output:
Chunk ID: f52cad
Wall time: 5.0009 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: 5cf500
Wall time: 5.0008 seconds
Process running with session ID 23321
Original token count: 0
Output:


Output:
Chunk ID: a09f75
Wall time: 5.0009 seconds
Process running with session ID 23321
Original token count: 0
Output:


**Diagnosing deadlock from worker cancellation**


I’ve identified a likely issue in the new scale-down path: canceling extra workers can strand in-flight decode futures and force long timeouts. I’m stopping this run, patching to scale-up-only (no worker cancellation), then rerunning the same seeded test.


pkill -f "stress_test_streaming_runtime.py --levels 96,128 --text short --max-tokens 1024 --seed-base 2026 --timeout 300 --requests-multiplier 1 --min-requests 8 --gpu-memory 0.25 --stream-output-kind delta --stream-decode-interval 48 --stream-window-size 128 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --bicodec-batch-workers 2 --bicodec-batch-scale-pending 56" || true


Output:
Chunk ID: 2a621f
Wall time: 0.0508 seconds
Process exited with code -1
Original token count: 0
Output:


Output:
Chunk ID: a4a049
Wall time: 0.0505 seconds
Process exited with code 0
Original token count: 58
Output:
1:    PID     ELAPSED CMD
472: 109030       00:00 /bin/bash -c ps -eo pid,etime,cmd | rg -n "stress_test_streaming_runtime.py --levels 96,128|PID"
474: 109033       00:00 rg -n stress_test_streaming_runtime.py --levels 96,128|PID


**Simplifying worker target management**


Input:
*** Begin Patch
*** Update File: veena3modal/core/bicodec_decoder.py
@@
-            self._batch_workers_target_last = target_workers
-
             while len(live_workers) < target_workers:
                 worker_idx = len(live_workers)
                 worker = loop.create_task(
                     self._batch_worker(loop, queue, worker_idx),
                     name=f"bicodec-batch-worker-{key}-{worker_idx}",
                 )
                 live_workers.append(worker)

-            while len(live_workers) > target_workers:
-                worker = live_workers.pop()
-                worker.cancel()
-
+            self._batch_workers_target_last = len(live_workers)
             self._batch_workers[key] = live_workers
             return queue
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/bicodec_decoder.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Confirming compilation step**


python3 -m py_compile veena3modal/core/bicodec_decoder.py


Output:
Chunk ID: 5eba64
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 0
Output:


**Adjusting command timeouts**


I patched the scaler to be scale-up-only (no worker cancellation), which avoids stranded decode futures. I’m rerunning the same seeded `max_workers=2, scale_pending=56` benchmark now.


python3 scripts/stress_test_streaming_runtime.py --levels 96,128 --text short --max-tokens 1024 --seed-base 2026 --timeout 240 --requests-multiplier 1 --min-requests 8 --gpu-memory 0.25 --stream-output-kind delta --stream-decode-interval 48 --stream-window-size 128 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --bicodec-batch-workers 2 --bicodec-batch-scale-pending 56 --output /tmp/stream_workers2_scale56_seeded.json


Output:
Chunk ID: 3fe93b
Wall time: 1.0011 seconds
Process running with session ID 43707
Original token count: 0
Output:


Output:
Chunk ID: 0d9eb0
Wall time: 5.0016 seconds
Process running with session ID 43707
Original token count: 1400
Output:
Total output lines: 52

[2026-02-18 18:52:03] INFO tts_runtime.py:183: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.25
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-18 18:52:03 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-18 18:52:03 [model.py:1561] Using max model len 4096
INFO 02-18 18:52:03 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-18 18:52:03 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:04 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype…200 tokens truncated…alse, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:07 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:53139 backend=nccl
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:07 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:07 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...


Output:
Chunk ID: 8f497c
Wall time: 5.0009 seconds
Process running with session ID 43707
Original token count: 384
Output:
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:08 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=109528) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=109528) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=109528)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=109528)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.06it/s]
(EngineCore_DP0 pid=109528)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.06it/s]
(EngineCore_DP0 pid=109528)
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:09 [default_loader.py:291] Loading weights took 1.02 seconds
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:10 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.540450 seconds
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:14 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:14 [backends.py:872] Dynamo bytecode transform time: 4.22 s


Output:
Chunk ID: 346dbb
Wall time: 5.0007 seconds
Process running with session ID 43707
Original token count: 785
Output:
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:17 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.651 s
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:17 [monitor.py:34] torch.compile takes 4.87 s in total
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:18 [gpu_worker.py:356] Available KV cache memory: 18.06 GiB
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:18 [kv_cache_utils.py:1307] GPU KV cache size: 1,577,712 tokens
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:18 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 385.18x
(EngineCore_DP0 pid=109528)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 33.12it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 35.51it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 36.47it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 36.65it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 36.28it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 36.72it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 36.56it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 34.65it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:01<00:00, 34.82it/s]
(EngineCore_DP0 pid=109528)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 29.17it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 33.70it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 34.92it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 35.35it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.95it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.09it/s]
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:20 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=109528) INFO 02-18 18:52:20 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.55 seconds
[2026-02-18 18:52:22] INFO tts_runtime.py:192: Initializing prompt builder...
[2026-02-18 18:52:22] INFO tts_runtime.py:199: Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


Output:
Chunk ID: ee255e
Wall time: 5.0006 seconds
Process running with session ID 43707
Original token count: 127
Output:
[2026-02-18 18:52:28] INFO tts_runtime.py:209: Initializing TTS pipeline...
[2026-02-18 18:52:28] INFO tts_runtime.py:220: Initializing streaming pipeline...
[2026-02-18 18:52:28] INFO long_text_processor.py:112: LongTextProcessor initialized
[2026-02-18 18:52:28] INFO tts_runtime.py:277: ✅ TTS runtime initialized in 31327ms
[2026-02-18 18:52:28] INFO tts_runtime.py:278:    Model version: LLM
[2026-02-18 18:52:28] INFO tts_runtime.py:289: Speaker globals cache startup precompute skipped (disabled)


Output:
Chunk ID: 43a3d4
Wall time: 3.5126 seconds
Process exited with code 0
Original token count: 1240
Output:
Total output lines: 87

✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cuda
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
   ℹ  Micro-batching enabled (workers=2, scale_pending=56, max_batch=16, timeout_ms=6.0)
🚀 SparkTTSPipeline initialized
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)

=== streaming concurrency=96 requests=96 ===
success=96/96 (100%)
wall=4.34s throughput=22.14 req/s
latency_ms avg=3743 p50=4038 p95=4334 max=4334
ttfb_ms avg=1133 p50=1070 p95=1451 max=1451
chunks_per_req avg=5.1 p50=5.0 p95=6.0 max=6.0
inter_chunk_ms avg=628.2 p50=555.4 p95=1816.3 max=1937.4
tail_emit_speed_x avg=1.49 p50=1.43 p95=2.01 max=2.21
timeline_to_first_batch_ms: avg=54.07 p50=54.40 p95=60.38 max=62.15
timeline_to_first_chunk_emitted_ms: avg=1113.36 p50=1059.22 p95=1415.35 max=1416.59
timeline_total_ms: avg=3672.13 p50=3899.39 p95=4301.42 max=4305.89
llm_time_in_queue_ms: avg=0.10 p50=0.08 p95=0.25 max=0.28
llm_scheduler_ms: avg=38.00 p50=38.95 p95=44.96 max=44.96
llm_first_token_ms: avg=51.52 p50=51.78 p95=58.60 max=59.91
llm_request_lifecycle_ms: avg=2514.66 p50=2541.98 p95=2701.82 max=2819.76
llm_generation_wall_ms: avg=3653.33 p50=3886.74 p95=4301.38 max=4305.85
llm_time_per_token_ms: avg=16.12 p50=16.66 p95=18.57 max=18.89
llm_batch_wall_ms_total: avg=3245.40 p50=3207.73 p95=3998.84 max=4191.66
llm_decode_wall_ms_total: avg=3191.36 p50=3155.79 p95=3942.70 max=4136.01
tokens_per_batch_p50: avg=1.00 p50=1.00 p95=1.00 max=1.00
bicodec_decode_calls: avg=4.07 p50=4.00 p95=5.00 max=5.00
bicodec_decode_wall_ms_total: avg=2296.51 p50=2339.94 p95=3515.63 max=3529.99
bicodec_decode_interval_ms_p50: avg=664.37 p50=622.72 p95=946.39 max=1093.20
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
batch_avg_size: avg=9.70 p50=8.82 p95=12.20 max=12.20
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_workers_live: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_workers_target: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_queue_wait_ms_avg: avg=376.69 p50=431.18 p95=471.88 max=489.25
batch_compute_ms_avg: avg=184.62 p50=172.23 p…40 tokens truncated…e_share: llm=99.5% bicodec=62.5% other=0.0%
gpu: util_avg=77% util_max=100% mem_avg=47572MB mem_max=48699MB

=== streaming concurrency=128 requests=128 ===
success=128/128 (100%)
wall=5.48s throughput=23.37 req/s
latency_ms avg=4549 p50=4647 p95=5473 max=5473
ttfb_ms avg=1363 p50=1318 p95=1644 max=1645
chunks_per_req avg=5.0 p50=5.0 p95=5.0 max=6.0
inter_chunk_ms avg=783.1 p50=786.0 p95=2109.8 max=2446.5
tail_emit_speed_x avg=1.25 p50=1.18 p95=1.83 max=2.84
timeline_to_first_batch_ms: avg=65.68 p50=66.65 p95=77.44 max=79.92
timeline_to_first_chunk_emitted_ms: avg=1336.69 p50=1293.30 p95=1599.92 max=1604.05
timeline_total_ms: avg=4499.90 p50=4607.82 p95=5436.31 max=5439.95
llm_time_in_queue_ms: avg=0.21 p50=0.11 p95=0.54 max=0.58
llm_scheduler_ms: avg=44.57 p50=40.81 p95=59.20 max=59.20
llm_first_token_ms: avg=62.35 p50=63.89 p95=74.66 max=76.86
llm_request_lifecycle_ms: avg=2932.09 p50=2940.77 p95=3180.07 max=3274.85
llm_generation_wall_ms: avg=4481.22 p50=4579.20 p95=5436.27 max=5439.91
llm_time_per_token_ms: avg=19.82 p50=20.71 p95=23.17 max=24.64
llm_batch_wall_ms_total: avg=4095.49 p50=4064.57 p95=5054.00 max=5294.36
llm_decode_wall_ms_total: avg=4029.84 p50=3997.11 p95=4995.51 max=5224.40
tokens_per_batch_p50: avg=1.00 p50=1.00 p95=1.00 max=1.00
bicodec_decode_calls: avg=4.04 p50=4.00 p95=4.00 max=5.00
bicodec_decode_wall_ms_total: avg=3237.15 p50=3530.56 p95=4487.92 max=4564.38
bicodec_decode_interval_ms_p50: avg=990.38 p50=983.21 p95=1579.33 max=1646.22
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
batch_avg_size: avg=7.90 p50=7.91 p95=9.34 max=9.95
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_workers_live: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_workers_target: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_queue_wait_ms_avg: avg=563.68 p50=567.73 p95=671.48 max=671.48
batch_compute_ms_avg: avg=134.09 p50=135.42 p95=156.80 max=163.02
batch_queue_depth_avg: avg=59.39 p50=60.19 p95=62.11 max=62.14
latency_over_timeline_ms: avg=49.09 p50=37.32 p95=155.97 max=216.83
timeline_share: llm=99.6% bicodec=71.9% other=0.0%
gpu: util_avg=71% util_max=99% mem_avg=49521MB mem_max=49873MB

results saved: /tmp/stream_workers2_scale56_seeded.json
ERROR 02-18 18:52:42 [core_client.py:605] Engine core proc EngineCore_DP0 died unexpectedly, shutting down client.


**Noting increased GPU memory usage**


python3 - <<'PY'
import json
profiles={
 'w1':'/tmp/stream_workers1_seeded.json',
 'w2':'/tmp/stream_workers2_seeded.json',
 'w2_scale56':'/tmp/stream_workers2_scale56_seeded.json',
}
res={}
for k,p in profiles.items():
    d=json.load(open(p))['levels']
    res[k]={r['concurrency']:r for r in d}
for c in [96,128]:
    print(f'\n=== c={c} ===')
    for k in ['w1','w2','w2_scale56']:
        r=res[k][c]; t=r['timing']
        print(f"{k:11s} thr={r['throughput_rps']:.2f} lat={r['latency_ms']['avg']:.0f} ttfb={r['ttfb_ms']['avg']:.0f} bicodec={t['bicodec_decode_wall_ms_total']['avg']:.0f} qwait={t['batch_queue_wait_ms_avg']['avg']:.0f} qdepth={t['batch_queue_depth_avg']['avg']:.1f} batch_avg={t['batch_avg_size']['avg']:.2f} workers={t.get('batch_workers_live',{}).get('avg',0):.1f}")
PY


Output:
Chunk ID: e3dbe7
Wall time: 0.0501 seconds
Process exited with code 0
Original token count: 162
Output:

=== c=96 ===
w1          thr=23.05 lat=3627 ttfb=1155 bicodec=2344 qwait=478 qdepth=45.3 batch_avg=10.03 workers=1.0
w2          thr=22.25 lat=3763 ttfb=1151 bicodec=2677 qwait=502 qdepth=48.0 batch_avg=8.46 workers=2.0
w2_scale56  thr=22.14 lat=3743 ttfb=1133 bicodec=2297 qwait=377 qdepth=41.0 batch_avg=9.70 workers=2.0

=== c=128 ===
w1          thr=22.79 lat=4619 ttfb=1520 bicodec=3567 qwait=694 qdepth=68.4 batch_avg=8.52 workers=1.0
w2          thr=23.12 lat=4565 ttfb=1383 bicodec=2979 qwait=576 qdepth=57.0 batch_avg=8.53 workers=2.0
w2_scale56  thr=23.37 lat=4549 ttfb=1363 bicodec=3237 qwait=564 qdepth=59.4 batch_avg=7.90 workers=2.0


**Testing scale-up threshold adjustments**


`scale_pending=56` still activates 2 workers at `c=96` in this workload. I’m running one more seeded pass with `scale_pending=64` to try keeping `c=96` on 1 worker while still triggering extra capacity at `c=128`.


python3 scripts/stress_test_streaming_runtime.py --levels 96,128 --text short --max-tokens 1024 --seed-base 2026 --timeout 240 --requests-multiplier 1 --min-requests 8 --gpu-memory 0.25 --stream-output-kind delta --stream-decode-interval 48 --stream-window-size 128 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --bicodec-batch-workers 2 --bicodec-batch-scale-pending 64 --output /tmp/stream_workers2_scale64_seeded.json


Output:
Chunk ID: ef37f3
Wall time: 1.0016 seconds
Process running with session ID 30703
Original token count: 0
Output:


Summary: auto


Output:
Chunk ID: 2ba2e9
Wall time: 5.0015 seconds
Process running with session ID 30703
Original token count: 1587
Output:
Total output lines: 56

[2026-02-18 18:53:09] INFO tts_runtime.py:183: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.25
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-18 18:53:10 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-18 18:53:10 [model.py:1561] Using max model len 4096
INFO 02-18 18:53:10 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-18 18:53:10 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:11 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype…387 tokens truncated…'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:13 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:34173 backend=nccl
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:13 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:13 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:14 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=110730) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=110730) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=110730)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]


Output:
Chunk ID: 66fa84
Wall time: 5.0011 seconds
Process running with session ID 30703
Original token count: 197
Output:
(EngineCore_DP0 pid=110730)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.26s/it]
(EngineCore_DP0 pid=110730)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.26s/it]
(EngineCore_DP0 pid=110730)
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:16 [default_loader.py:291] Loading weights took 1.34 seconds
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:16 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.889396 seconds
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:21 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:21 [backends.py:872] Dynamo bytecode transform time: 4.24 s


Output:
Chunk ID: 097f07
Wall time: 5.0007 seconds
Process running with session ID 30703
Original token count: 785
Output:
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:24 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.629 s
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:24 [monitor.py:34] torch.compile takes 4.87 s in total
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:25 [gpu_worker.py:356] Available KV cache memory: 18.06 GiB
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:25 [kv_cache_utils.py:1307] GPU KV cache size: 1,577,712 tokens
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:25 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 385.18x
(EngineCore_DP0 pid=110730)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 32.27it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 34.41it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 34.84it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 35.56it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 35.12it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 35.90it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 35.80it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 33.82it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:01<00:00, 33.99it/s]
(EngineCore_DP0 pid=110730)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 28.81it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 32.95it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 34.07it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 34.99it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.22it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 34.43it/s]
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:27 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=110730) INFO 02-18 18:53:27 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.84 seconds
[2026-02-18 18:53:29] INFO tts_runtime.py:192: Initializing prompt builder...
[2026-02-18 18:53:29] INFO tts_runtime.py:199: Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


The `scale_pending=64` run is progressing normally. I’ll report whether the second worker stayed off at `c=96` (target=1) and turned on at `c=128` (target=2), which is the behavior we want.


Output:
Chunk ID: 633d00
Wall time: 5.0003 seconds
Process running with session ID 30703
Original token count: 127
Output:
[2026-02-18 18:53:36] INFO tts_runtime.py:209: Initializing TTS pipeline...
[2026-02-18 18:53:36] INFO tts_runtime.py:220: Initializing streaming pipeline...
[2026-02-18 18:53:36] INFO long_text_processor.py:112: LongTextProcessor initialized
[2026-02-18 18:53:36] INFO tts_runtime.py:277: ✅ TTS runtime initialized in 32364ms
[2026-02-18 18:53:36] INFO tts_runtime.py:278:    Model version: LLM
[2026-02-18 18:53:36] INFO tts_runtime.py:289: Speaker globals cache startup precompute skipped (disabled)


Output:
Chunk ID: b091ce
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 1241
Output:
Total output lines: 87

✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cuda
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
   ℹ  Micro-batching enabled (workers=2, scale_pending=64, max_batch=16, timeout_ms=6.0)
🚀 SparkTTSPipeline initialized
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)

=== streaming concurrency=96 requests=96 ===
success=96/96 (100%)
wall=4.36s throughput=22.01 req/s
latency_ms avg=3710 p50=3733 p95=4326 max=4360
ttfb_ms avg=1115 p50=1079 p95=1304 max=1304
chunks_per_req avg=5.2 p50=5.0 p95=6.0 max=6.0
inter_chunk_ms avg=605.4 p50=577.9 p95=1751.0 max=1866.6
tail_emit_speed_x avg=1.54 p50=1.46 p95=2.22 max=2.47
timeline_to_first_batch_ms: avg=61.40 p50=62.30 p95=70.01 max=71.95
timeline_to_first_chunk_emitted_ms: avg=1096.07 p50=1068.79 p95=1270.56 max=1271.66
timeline_total_ms: avg=3624.76 p50=3729.66 p95=4298.83 max=4325.89
llm_time_in_queue_ms: avg=0.15 p50=0.13 p95=0.33 max=0.36
llm_scheduler_ms: avg=40.20 p50=32.10 p95=55.29 max=55.29
llm_first_token_ms: avg=58.64 p50=59.54 p95=68.87 max=70.93
llm_request_lifecycle_ms: avg=2461.46 p50=2485.88 p95=2678.42 max=2726.43
llm_generation_wall_ms: avg=3605.74 p50=3729.61 p95=4298.79 max=4325.85
llm_time_per_token_ms: avg=15.70 p50=16.03 p95=18.50 max=18.77
llm_batch_wall_ms_total: avg=3274.07 p50=3296.84 p95=4063.35 max=4196.17
llm_decode_wall_ms_total: avg=3212.70 p50=3237.80 p95=4001.34 max=4131.67
tokens_per_batch_p50: avg=1.00 p50=1.00 p95=1.00 max=1.00
bicodec_decode_calls: avg=4.18 p50=4.00 p95=5.00 max=5.00
bicodec_decode_wall_ms_total: avg=2258.26 p50=2276.16 p95=3593.07 max=3627.32
bicodec_decode_interval_ms_p50: avg=657.52 p50=643.07 p95=1007.90 max=1159.34
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
batch_avg_size: avg=9.58 p50=8.94 p95=11.57 max=11.57
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_workers_live: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_workers_target: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_queue_wait_ms_avg: avg=371.78 p50=416.32 p95=474.64 max=487.74
batch_compute_ms_avg: avg=171.20 p50=164.75 …41 tokens truncated…_share: llm=99.5% bicodec=62.3% other=0.0%
gpu: util_avg=71% util_max=99% mem_avg=47453MB mem_max=49213MB

=== streaming concurrency=128 requests=128 ===
success=128/128 (100%)
wall=4.77s throughput=26.81 req/s
latency_ms avg=4217 p50=4535 p95=4770 max=4770
ttfb_ms avg=1227 p50=1237 p95=1540 max=1546
chunks_per_req avg=5.0 p50=5.0 p95=5.0 max=6.0
inter_chunk_ms avg=735.5 p50=787.5 p95=1813.2 max=2016.9
tail_emit_speed_x avg=1.30 p50=1.24 p95=1.63 max=1.79
timeline_to_first_batch_ms: avg=62.04 p50=62.71 p95=73.02 max=75.42
timeline_to_first_chunk_emitted_ms: avg=1201.47 p50=1214.32 p95=1495.76 max=1500.00
timeline_total_ms: avg=4154.81 p50=4511.94 p95=4731.20 max=4734.99
llm_time_in_queue_ms: avg=0.15 p50=0.13 p95=0.34 max=0.37
llm_scheduler_ms: avg=41.96 p50=38.98 p95=55.67 max=55.67
llm_first_token_ms: avg=58.85 p50=60.19 p95=70.55 max=72.39
llm_request_lifecycle_ms: avg=2982.64 p50=2995.13 p95=3236.84 max=3431.38
llm_generation_wall_ms: avg=4142.52 p50=4503.75 p95=4731.16 max=4734.95
llm_time_per_token_ms: avg=18.18 p50=18.67 p95=20.56 max=22.40
llm_batch_wall_ms_total: avg=3700.35 p50=3732.77 p95=4315.00 max=4517.16
llm_decode_wall_ms_total: avg=3638.34 p50=3667.66 p95=4259.25 max=4468.44
tokens_per_batch_p50: avg=1.00 p50=1.00 p95=1.00 max=1.00
bicodec_decode_calls: avg=4.02 p50=4.00 p95=4.00 max=5.00
bicodec_decode_wall_ms_total: avg=2473.44 p50=2775.08 p95=3713.63 max=3866.10
bicodec_decode_interval_ms_p50: avg=878.30 p50=879.67 p95=1075.17 max=1162.09
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
batch_avg_size: avg=9.32 p50=9.09 p95=10.09 max=10.09
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_workers_live: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_workers_target: avg=2.00 p50=2.00 p95=2.00 max=2.00
batch_queue_wait_ms_avg: avg=463.16 p50=490.93 p95=518.00 max=518.00
batch_compute_ms_avg: avg=149.42 p50=147.12 p95=155.99 max=164.41
batch_queue_depth_avg: avg=48.03 p50=49.19 p95=50.27 max=50.27
latency_over_timeline_ms: avg=62.66 p50=37.82 p95=189.93 max=336.59
timeline_share: llm=99.7% bicodec=59.5% other=0.0%
gpu: util_avg=75% util_max=98% mem_avg=49213MB mem_max=49213MB

results saved: /tmp/stream_workers2_scale64_seeded.json
ERROR 02-18 18:53:49 [core_client.py:605] Engine core proc EngineCore_DP0 died unexpectedly, shutting down client.


**Testing queue threshold tuning**


`scale_pending=64` delivered a major `c=128` gain (`26.81 req/s`), but still flips to 2 workers at `c=96`. I’m running one more pass at `scale_pending=72` to see if we can preserve the `c=128` gain while keeping `c=96` closer to the 1-worker baseline.


python3 scripts/stress_test_streaming_runtime.py --levels 96,128 --text short --max-tokens 1024 --seed-base 2026 --timeout 240 --requests-multiplier 1 --min-requests 8 --gpu-memory 0.25 --stream-output-kind delta --stream-decode-interval 48 --stream-window-size 128 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --bicodec-batch-workers 2 --bicodec-batch-scale-pending 72 --output /tmp/stream_workers2_scale72_seeded.json


Output:
Chunk ID: f081e5
Wall time: 1.0012 seconds
Process running with session ID 84421
Original token count: 0
Output:


**Planning poll and update flow**