Input:
*** Begin Patch
*** Update File: veena3modal/core/bicodec_decoder.py
@@
         self._batch_total_batches = 0
         self._batch_total_requests = 0
         self._batch_max_seen = 0
+        self._batch_queue_wait_ms_total = 0.0
+        self._batch_queue_wait_ms_min: Optional[float] = None
+        self._batch_queue_wait_ms_max = 0.0
+        self._batch_compute_ms_total = 0.0
+        self._batch_compute_ms_min: Optional[float] = None
+        self._batch_compute_ms_max = 0.0
+        self._batch_queue_depth_total = 0
+        self._batch_queue_depth_max = 0
+        self._batch_inflight_requests = 0
@@
                 batch_inputs = [
                     (req.semantic_ids, req.global_ids, req.use_sliding_window, req.trim_warmup)
                     for req in batch
                 ]
-                t0 = time.perf_counter()
+                batch_start = time.perf_counter()
+                queue_wait_ms = [
+                    max(0.0, (batch_start - req.submitted_at) * 1000.0)
+                    for req in batch
+                ]
+                queue_depth_now = queue.qsize()
+                self._batch_queue_depth_total += queue_depth_now
+                if queue_depth_now > self._batch_queue_depth_max:
+                    self._batch_queue_depth_max = queue_depth_now
+                for wait_ms in queue_wait_ms:
+                    self._batch_queue_wait_ms_total += wait_ms
+                    if self._batch_queue_wait_ms_min is None or wait_ms < self._batch_queue_wait_ms_min:
+                        self._batch_queue_wait_ms_min = wait_ms
+                    if wait_ms > self._batch_queue_wait_ms_max:
+                        self._batch_queue_wait_ms_max = wait_ms
+                self._batch_inflight_requests += len(batch)
+
+                t0 = time.perf_counter()
                 try:
                     results = await loop.run_in_executor(None, self._decode_streaming_batch_sync, batch_inputs)
                     if len(results) != len(batch):
                         raise RuntimeError("batch decode result size mismatch")
                     for req, result in zip(batch, results):
@@
                 except Exception as exc:
                     for req in batch:
                         if not req.future.done():
                             req.future.set_exception(exc)
                 finally:
-                    _ = (time.perf_counter() - t0) * 1000.0
+                    compute_ms = (time.perf_counter() - t0) * 1000.0
+                    self._batch_compute_ms_total += compute_ms
+                    if self._batch_compute_ms_min is None or compute_ms < self._batch_compute_ms_min:
+                        self._batch_compute_ms_min = compute_ms
+                    if compute_ms > self._batch_compute_ms_max:
+                        self._batch_compute_ms_max = compute_ms
                     self._batch_total_batches += 1
                     self._batch_total_requests += len(batch)
                     if len(batch) > self._batch_max_seen:
                         self._batch_max_seen = len(batch)
+                    self._batch_inflight_requests = max(0, self._batch_inflight_requests - len(batch))
         finally:
             with self._batch_map_lock:
                 self._batch_queues.pop(key, None)
                 self._batch_workers.pop(key, None)

+    def get_pending_requests(self) -> int:
+        if not self.enable_batching:
+            return 0
+        with self._batch_map_lock:
+            pending = sum(queue.qsize() for queue in self._batch_queues.values())
+            return int(max(0, pending + self._batch_inflight_requests))
+
     def get_batching_stats(self) -> Dict[str, float]:
         batches = max(1, self._batch_total_batches)
+        reqs = max(1, self._batch_total_requests)
         return {
             "enabled": 1.0 if self.enable_batching else 0.0,
             "batch_total_batches": float(self._batch_total_batches),
             "batch_total_requests": float(self._batch_total_requests),
             "batch_avg_size": float(self._batch_total_requests / batches),
             "batch_max_seen": float(self._batch_max_seen),
+            "batch_queue_wait_ms_avg": float(self._batch_queue_wait_ms_total / reqs),
+            "batch_queue_wait_ms_min": float(self._batch_queue_wait_ms_min or 0.0),
+            "batch_queue_wait_ms_max": float(self._batch_queue_wait_ms_max),
+            "batch_compute_ms_avg": float(self._batch_compute_ms_total / batches),
+            "batch_compute_ms_min": float(self._batch_compute_ms_min or 0.0),
+            "batch_compute_ms_max": float(self._batch_compute_ms_max),
+            "batch_queue_depth_avg": float(self._batch_queue_depth_total / batches),
+            "batch_queue_depth_max": float(self._batch_queue_depth_max),
+            "batch_pending_now": float(self.get_pending_requests()),
         }
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/bicodec_decoder.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Adding locks for thread safety**


Input:
*** Begin Patch
*** Update File: veena3modal/core/bicodec_decoder.py
@@
-                self._batch_inflight_requests += len(batch)
+                with self._batch_map_lock:
+                    self._batch_inflight_requests += len(batch)
@@
-                    self._batch_inflight_requests = max(0, self._batch_inflight_requests - len(batch))
+                    with self._batch_map_lock:
+                        self._batch_inflight_requests = max(0, self._batch_inflight_requests - len(batch))
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/bicodec_decoder.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Summary: auto


**Planning incremental patch insertion**


Input:
*** Begin Patch
*** Update File: veena3modal/core/streaming_pipeline.py
@@
 def _env_int(name: str, default: int, minimum: int = 0) -> int:
     raw = os.environ.get(name, "")
     if not raw:
         return default
@@
     return max(minimum, value)

+def _env_flag(name: str, default: bool) -> bool:
+    raw = os.environ.get(name, "")
+    if not raw:
+        return default
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
 _STREAM_EXPECTED_GLOBAL_COUNT = 32
 _STREAM_MIN_SEMANTIC_FIRST_CHUNK = _env_int("VEENA3_STREAM_MIN_SEMANTIC_FIRST", 10, minimum=1)
 _STREAM_DECODE_INTERVAL = _env_int("VEENA3_STREAM_DECODE_INTERVAL", 48, minimum=1)
 _STREAM_CROSSFADE_MS = _env_int("VEENA3_STREAM_CROSSFADE_MS", 50, minimum=0)
 _STREAM_WINDOW_SIZE = _env_int("VEENA3_STREAM_WINDOW_SIZE", 128, minimum=16)
+_STREAM_ADAPTIVE_DECODE = _env_flag("VEENA3_STREAM_ADAPTIVE_DECODE", False)
+_STREAM_DECODE_INTERVAL_FIRST = _env_int("VEENA3_STREAM_DECODE_INTERVAL_FIRST", _STREAM_DECODE_INTERVAL, minimum=1)
+_STREAM_DECODE_INTERVAL_BUSY = _env_int(
+    "VEENA3_STREAM_DECODE_INTERVAL_BUSY",
+    max(_STREAM_DECODE_INTERVAL, 64),
+    minimum=1,
+)
+_STREAM_DECODE_BUSY_PENDING = _env_int("VEENA3_STREAM_DECODE_BUSY_PENDING", 32, minimum=1)
 _STREAM_WINDOWED_DECODE = os.environ.get("VEENA3_STREAM_WINDOWED_DECODE", "true").strip().lower() in {
     "1",
     "true",
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Adding helper methods after stats patch**


Input:
*** Begin Patch
*** Update File: veena3modal/core/streaming_pipeline.py
@@
     def _apply_stats(self, perf: Dict[str, Any], prefix: str, values: List[float]) -> None:
         if not values:
             return
         perf[f"{prefix}_total"] = float(sum(values))
         perf[f"{prefix}_min"] = float(min(values))
         perf[f"{prefix}_max"] = float(max(values))
         perf[f"{prefix}_p50"] = float(self._p50(values))
+
+    def _get_decoder_pending_requests(self) -> int:
+        getter = getattr(self.snac_decoder, "get_pending_requests", None)
+        if not callable(getter):
+            return 0
+        try:
+            return max(0, int(getter()))
+        except Exception:
+            return 0
+
+    def _resolve_decode_interval(self, first_chunk_emitted: bool) -> Tuple[int, int]:
+        interval = int(_STREAM_DECODE_INTERVAL)
+        pending = 0
+        if not _STREAM_ADAPTIVE_DECODE:
+            return interval, pending
+
+        if not first_chunk_emitted:
+            interval = int(_STREAM_DECODE_INTERVAL_FIRST)
+
+        pending = self._get_decoder_pending_requests()
+        if pending >= _STREAM_DECODE_BUSY_PENDING:
+            interval = max(interval, int(_STREAM_DECODE_INTERVAL_BUSY))
+        return max(1, interval), pending
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Planning decode loop patch**


Input:
*** Begin Patch
*** Update File: veena3modal/core/streaming_pipeline.py
@@
         tokens_per_batch: List[float] = []
         bicodec_decode_wall_ms: List[float] = []
         bicodec_decode_tokens: List[float] = []
         bicodec_decode_intervals_ms: List[float] = []
+        decode_interval_applied_tokens: List[float] = []
+        decode_pending_requests: List[float] = []
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Preparing targeted condition patch**


sed -n '468,555p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: fd22ad
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1073
Output:

            outputs = request_output.outputs or []
            if not outputs:
                continue

            generated_ids = outputs[0].token_ids
            if _STREAM_USE_DELTA_OUTPUT:
                new_token_ids = generated_ids
                processed_token_count += len(new_token_ids)
            else:
                new_token_ids = generated_ids[processed_token_count:]
                processed_token_count = len(generated_ids)

            new_token_count = len(new_token_ids)
            if new_token_count <= 0:
                continue

            total_generated_tokens += new_token_count
            llm_batch_wall_ms.append(float(batch_wall))
            tokens_per_batch.append(float(new_token_count))

            if "llm_first_batch" not in mark_stage_set:
                _mark("llm_first_batch")
            elif llm_batch_wall_ms:
                llm_decode_wall_ms.append(float(batch_wall))

            raw_req_metrics = self._extract_request_metrics(getattr(request_output, "metrics", None))
            if raw_req_metrics:
                final_request_metrics = raw_req_metrics
                cumulative_gpu_s: Optional[float] = None
                if "model_execute_time" in raw_req_metrics:
                    cumulative_gpu_s = raw_req_metrics["model_execute_time"]
                elif "model_forward_time" in raw_req_metrics:
                    cumulative_gpu_s = raw_req_metrics["model_forward_time"]

                if cumulative_gpu_s is not None:
                    if prev_gpu_cumulative_s is None:
                        gpu_delta_ms = cumulative_gpu_s * 1000.0
                    elif cumulative_gpu_s >= prev_gpu_cumulative_s:
                        gpu_delta_ms = (cumulative_gpu_s - prev_gpu_cumulative_s) * 1000.0
                    else:
                        gpu_delta_ms = cumulative_gpu_s * 1000.0
                    prev_gpu_cumulative_s = cumulative_gpu_s
                    if gpu_delta_ms >= 0.0:
                        llm_batch_gpu_ms.append(float(gpu_delta_ms))
                        if len(llm_batch_wall_ms) > 1:
                            llm_decode_gpu_ms.append(float(gpu_delta_ms))

            parse_t0 = time.perf_counter()
            token_parser.parse_incremental(new_token_ids, semantic_buffer, global_buffer)
            llm_parse_ms_samples.append((time.perf_counter() - parse_t0) * 1000.0)

            if len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
                semantic_count = len(semantic_buffer)
                if semantic_count >= _STREAM_MIN_SEMANTIC_FIRST_CHUNK:
                    if "first_chunk_ready" not in mark_stage_set:
                        _mark("first_chunk_ready")

                    if semantic_count - last_decode_count >= _STREAM_DECODE_INTERVAL:
                        last_decode_count = semantic_count
                        decode_global = global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT]

                        if (
                            _STREAM_WINDOWED_DECODE
                            and semantic_count > _STREAM_WINDOW_SIZE
                        ):
                            decode_semantic = semantic_buffer[-_STREAM_WINDOW_SIZE:]
                        else:
                            decode_semantic = semantic_buffer

                        decode_t0 = time.perf_counter()
                        audio_bytes = await self.snac_decoder.decode_single_async(
                            semantic_ids=decode_semantic,
                            global_ids=decode_global,
                            trim_warmup=False,
                            use_sliding_window=False,
                        )
                        decode_t1 = time.perf_counter()

                        decode_wall_ms = (decode_t1 - decode_t0) * 1000.0
                        bicodec_decode_wall_ms.append(float(decode_wall_ms))
                        bicodec_decode_tokens.append(float(len(decode_semantic)))
                        if last_decode_end_ts is not None:
                            bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
                        last_decode_end_ts = decode_t1

                        if audio_bytes:
                            total_samples_decoded = len(audio_bytes) // 2


sed -n '555,625p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: 5e72b7
Wall time: 0.0503 seconds
Process exited with code 0
Original token count: 995
Output:
                            total_samples_decoded = len(audio_bytes) // 2
                            if (
                                _STREAM_WINDOWED_DECODE
                                and semantic_count > _STREAM_WINDOW_SIZE
                            ):
                                new_samples_approx = _STREAM_DECODE_INTERVAL * 320
                                if total_samples_decoded > new_samples_approx:
                                    new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
                                else:
                                    new_bytes_start = 0
                                new_audio_bytes = audio_bytes[new_bytes_start:]
                            else:
                                if total_samples_decoded > total_samples_emitted_to_user:
                                    new_bytes_start = total_samples_emitted_to_user * 2
                                    new_audio_bytes = audio_bytes[new_bytes_start:]
                                else:
                                    new_audio_bytes = b""

                            if new_audio_bytes:
                                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                                    previous_chunk_tail,
                                    new_audio_bytes,
                                    sample_rate_hz=16000,
                                    crossfade_ms=_STREAM_CROSSFADE_MS,
                                )
                                if to_emit:
                                    total_audio_chunks += 1
                                    total_samples_emitted_to_user += len(to_emit) // 2
                                    if t_first_chunk_decode_done is None:
                                        t_first_chunk_decode_done = decode_t1
                                        _mark("first_chunk_decoded")
                                    if t_first_chunk_emitted is None:
                                        t_first_chunk_emitted = time.perf_counter()
                                        _mark("first_chunk_emitted")
                                    if emit_progress:
                                        yield (to_emit, {})
                                    else:
                                        yield to_emit

        llm_generation_wall_ms = (time.perf_counter() - t_llm_start) * 1000.0
        _mark("llm_done")

        if len(semantic_buffer) > last_decode_count and len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
            decode_semantic = semantic_buffer
            decode_t0 = time.perf_counter()
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=decode_semantic,
                global_ids=global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT],
                use_sliding_window=False,
                trim_warmup=False,
            )
            decode_t1 = time.perf_counter()
            decode_wall_ms = (decode_t1 - decode_t0) * 1000.0
            bicodec_decode_wall_ms.append(float(decode_wall_ms))
            bicodec_decode_tokens.append(float(len(decode_semantic)))
            if last_decode_end_ts is not None:
                bicodec_decode_intervals_ms.append((decode_t1 - last_decode_end_ts) * 1000.0)
            last_decode_end_ts = decode_t1

            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2
                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=_STREAM_CROSSFADE_MS,
                    )
                    if to_emit:


**Preparing targeted patch update**


Input:
*** Begin Patch
*** Update File: veena3modal/core/streaming_pipeline.py
@@
             if len(global_buffer) >= _STREAM_EXPECTED_GLOBAL_COUNT:
                 semantic_count = len(semantic_buffer)
                 if semantic_count >= _STREAM_MIN_SEMANTIC_FIRST_CHUNK:
                     if "first_chunk_ready" not in mark_stage_set:
                         _mark("first_chunk_ready")

-                    if semantic_count - last_decode_count >= _STREAM_DECODE_INTERVAL:
+                    decode_interval, pending_requests = self._resolve_decode_interval(
+                        first_chunk_emitted=t_first_chunk_emitted is not None
+                    )
+                    semantic_delta = semantic_count - last_decode_count
+                    if semantic_delta >= decode_interval:
+                        decode_step_tokens = semantic_delta
                         last_decode_count = semantic_count
+                        decode_interval_applied_tokens.append(float(decode_interval))
+                        decode_pending_requests.append(float(pending_requests))
                         decode_global = global_buffer[:_STREAM_EXPECTED_GLOBAL_COUNT]

                         if (
                             _STREAM_WINDOWED_DECODE
                             and semantic_count > _STREAM_WINDOW_SIZE
@@
                         if audio_bytes:
                             total_samples_decoded = len(audio_bytes) // 2
                             if (
                                 _STREAM_WINDOWED_DECODE
                                 and semantic_count > _STREAM_WINDOW_SIZE
                             ):
-                                new_samples_approx = _STREAM_DECODE_INTERVAL * 320
+                                new_samples_approx = decode_step_tokens * 320
                                 if total_samples_decoded > new_samples_approx:
                                     new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
                                 else:
                                     new_bytes_start = 0
                                 new_audio_bytes = audio_bytes[new_bytes_start:]
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Scoping performance stats patch**


sed -n '635,705p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: f56527
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 924
Output:
                        if t_first_chunk_decode_done is None:
                            t_first_chunk_decode_done = decode_t1
                            _mark("first_chunk_decoded")
                        if t_first_chunk_emitted is None:
                            t_first_chunk_emitted = time.perf_counter()
                            _mark("first_chunk_emitted")
                        if emit_progress:
                            yield (to_emit, {})
                        else:
                            yield to_emit

        if previous_chunk_tail:
            total_audio_chunks += 1
            total_samples_emitted_to_user += len(previous_chunk_tail) // 2
            if t_first_chunk_emitted is None:
                t_first_chunk_emitted = time.perf_counter()
                _mark("first_chunk_emitted")
            if emit_progress:
                yield (previous_chunk_tail, {})
            else:
                yield previous_chunk_tail

        _mark("stream_done")
        _mark("request_done")

        perf: Dict[str, Any] = {
            "prompt_build_ms": float(prompt_build_ms),
            "sampling_params_ms": float(sampling_params_ms),
            "llm_generation_wall_ms": float(llm_generation_wall_ms),
            "llm_batch_count": int(len(llm_batch_wall_ms)),
            "llm_token_total": int(total_generated_tokens),
            "semantic_token_total": int(len(semantic_buffer)),
            "global_token_total": int(len(global_buffer)),
            "llm_parse_ms": float(sum(llm_parse_ms_samples)),
            "bicodec_decode_calls": int(len(bicodec_decode_wall_ms)),
            "llm_decode_calls": int(len(bicodec_decode_wall_ms)),
            "chunks_sent": int(total_audio_chunks),
            "audio_duration_seconds": float(total_samples_emitted_to_user / 16000.0),
            "ttfb_ms": float((t_first_chunk_emitted - t_request_start) * 1000.0 if t_first_chunk_emitted else 0.0),
            "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
            "stream_decode_interval_tokens": int(_STREAM_DECODE_INTERVAL),
            "stream_window_size_tokens": int(_STREAM_WINDOW_SIZE),
            "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
            "timeline_markers": list(timeline_marks),
            "timeline_total_ms": float((time.perf_counter() - t_request_start) * 1000.0),
        }

        decoder_batch_stats = {}
        if hasattr(self.snac_decoder, "get_batching_stats"):
            try:
                decoder_batch_stats = self.snac_decoder.get_batching_stats() or {}
            except Exception:
                decoder_batch_stats = {}
        if decoder_batch_stats:
            perf.update(decoder_batch_stats)

        self._apply_stats(perf, "llm_batch_wall_ms", llm_batch_wall_ms)
        self._apply_stats(perf, "llm_batch_gpu_ms", llm_batch_gpu_ms)
        self._apply_stats(perf, "llm_decode_wall_ms", llm_decode_wall_ms)
        self._apply_stats(perf, "llm_decode_gpu_ms", llm_decode_gpu_ms)
        self._apply_stats(perf, "tokens_per_batch", tokens_per_batch)
        self._apply_stats(perf, "llm_parse_step_ms", llm_parse_ms_samples)
        self._apply_stats(perf, "bicodec_decode_wall_ms", bicodec_decode_wall_ms)
        self._apply_stats(perf, "bicodec_decode_tokens", bicodec_decode_tokens)
        self._apply_stats(perf, "bicodec_decode_interval_ms", bicodec_decode_intervals_ms)

        perf["bicodec_decode_wall_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))
        perf["bicodec_decode_gpu_ms"] = 0.0
        perf["bicodec_decode_cpu_ms"] = float(perf.get("bicodec_decode_wall_ms_total", 0.0))

        if total_generated_tokens > 0:


**Patching block to add fields and stats**


Input:
*** Begin Patch
*** Update File: veena3modal/core/streaming_pipeline.py
@@
             "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
             "stream_decode_interval_tokens": int(_STREAM_DECODE_INTERVAL),
+            "stream_decode_interval_first_tokens": int(_STREAM_DECODE_INTERVAL_FIRST),
+            "stream_decode_interval_busy_tokens": int(_STREAM_DECODE_INTERVAL_BUSY),
+            "stream_decode_busy_pending_threshold": int(_STREAM_DECODE_BUSY_PENDING),
+            "stream_adaptive_decode": bool(_STREAM_ADAPTIVE_DECODE),
             "stream_window_size_tokens": int(_STREAM_WINDOW_SIZE),
             "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
             "timeline_markers": list(timeline_marks),
             "timeline_total_ms": float((time.perf_counter() - t_request_start) * 1000.0),
         }
@@
         self._apply_stats(perf, "llm_parse_step_ms", llm_parse_ms_samples)
         self._apply_stats(perf, "bicodec_decode_wall_ms", bicodec_decode_wall_ms)
         self._apply_stats(perf, "bicodec_decode_tokens", bicodec_decode_tokens)
         self._apply_stats(perf, "bicodec_decode_interval_ms", bicodec_decode_intervals_ms)
+        self._apply_stats(perf, "decode_interval_applied_tokens", decode_interval_applied_tokens)
+        self._apply_stats(perf, "decode_pending_requests", decode_pending_requests)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Preparing to patch adaptive intervals**


Input:
*** Begin Patch
*** Update File: veena3modal/core/streaming_pipeline.py
@@
         previous_chunk_tail: Optional[bytes] = None
         t_first_chunk_emitted: Optional[float] = None
         decode_calls = 0
         decode_wall_ms_total = 0.0
+        decode_interval_applied_tokens: List[float] = []
+        decode_pending_requests: List[float] = []
@@
             semantic_count = len(semantic_buffer)
             if semantic_count >= MIN_SEMANTIC_FOR_FIRST_CHUNK:
-                if semantic_count - last_decode_count >= DECODE_INTERVAL:
+                decode_interval, pending_requests = self._resolve_decode_interval(
+                    first_chunk_emitted=t_first_chunk_emitted is not None
+                )
+                semantic_delta = semantic_count - last_decode_count
+                if semantic_delta >= decode_interval:
+                    decode_step_tokens = semantic_delta
                     last_decode_count = semantic_count
+                    decode_interval_applied_tokens.append(float(decode_interval))
+                    decode_pending_requests.append(float(pending_requests))
                     if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
                         decode_semantic = semantic_buffer[-WINDOW_SIZE:]
                     else:
                         decode_semantic = semantic_buffer
@@
                     if audio_bytes:
                         total_samples_decoded = len(audio_bytes) // 2
                         if _STREAM_WINDOWED_DECODE and semantic_count > WINDOW_SIZE:
-                            new_samples_approx = DECODE_INTERVAL * 320
+                            new_samples_approx = decode_step_tokens * 320
                             if total_samples_decoded > new_samples_approx:
                                 new_bytes_start = (total_samples_decoded - new_samples_approx) * 2
                             else:
                                 new_bytes_start = 0
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Planning adaptive perf dict update**


I’ve finished adding decoder queue-wait instrumentation and I’m now wiring adaptive decode-interval metrics into both streaming paths so we can measure exactly when scheduling shifts under load.


**Continuing perf block patch near line 1180**


sed -n '1180,1235p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: d80ff4
Wall time: 0.0503 seconds
Process exited with code 0
Original token count: 611
Output:
                                if emit_progress:
                                    yield (to_emit, {})
                                else:
                                    yield to_emit

        # Final chunk processing
        if len(semantic_buffer) > last_decode_count:
            decode_t0 = time.perf_counter()
            audio_bytes = self.snac_decoder.decode_streaming(
                semantic_ids=semantic_buffer,
                global_ids=global_ids,  # Use captured globals
                use_sliding_window=False,
                trim_warmup=False
            )
            decode_calls += 1
            decode_wall_ms_total += (time.perf_counter() - decode_t0) * 1000.0
            if audio_bytes:
                total_samples_decoded = len(audio_bytes) // 2

                if total_samples_decoded > total_samples_emitted_to_user:
                    new_bytes_start = total_samples_emitted_to_user * 2
                    new_audio_bytes = audio_bytes[new_bytes_start:]

                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        new_audio_bytes,
                        sample_rate_hz=16000,
                        crossfade_ms=CROSSFADE_MS,
                    )

                    if to_emit:
                        total_audio_chunks += 1
                        total_samples_emitted_to_user += len(to_emit) // 2
                        if t_first_chunk_emitted is None:
                            t_first_chunk_emitted = time.perf_counter()
                        if emit_progress:
                            yield (to_emit, {})
                        else:
                            yield to_emit

        # Flush remaining tail
        if previous_chunk_tail:
            total_audio_chunks += 1
            tail_samples = len(previous_chunk_tail) // 2
            total_samples_emitted_to_user += tail_samples
            if t_first_chunk_emitted is None:
                t_first_chunk_emitted = time.perf_counter()
            if emit_progress:
                yield (previous_chunk_tail, {})
            else:
                yield previous_chunk_tail

        t_end = time.time()
        audio_duration_s = total_samples_emitted_to_user / 16000
        logger.debug(
            "continuation complete audio=%.2fs chunks=%d",


sed -n '1235,1288p' veena3modal/core/streaming_pipeline.py


Output:
Chunk ID: c7e911
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 410
Output:
            "continuation complete audio=%.2fs chunks=%d",
            audio_duration_s,
            total_audio_chunks,
        )

        if emit_progress:
            perf: Dict[str, Any] = {
                "llm_token_total": int(total_generated_tokens),
                "semantic_token_total": int(len(semantic_buffer)),
                "global_token_total": int(len(global_ids)),
                "bicodec_decode_calls": int(decode_calls),
                "llm_decode_calls": int(decode_calls),
                "bicodec_decode_wall_ms": float(decode_wall_ms_total),
                "bicodec_decode_gpu_ms": 0.0,
                "bicodec_decode_cpu_ms": float(decode_wall_ms_total),
                "chunks_sent": int(total_audio_chunks),
                "audio_duration_seconds": float(audio_duration_s),
                "ttfb_ms": float((t_first_chunk_emitted - t_start) * 1000.0 if t_first_chunk_emitted else 0.0),
                "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
                "stream_decode_interval_tokens": int(DECODE_INTERVAL),
                "stream_window_size_tokens": int(WINDOW_SIZE),
                "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
            }
            decoder_batch_stats = {}
            if hasattr(self.snac_decoder, "get_batching_stats"):
                try:
                    decoder_batch_stats = self.snac_decoder.get_batching_stats() or {}
                except Exception:
                    decoder_batch_stats = {}
            if decoder_batch_stats:
                perf.update(decoder_batch_stats)
            yield (b"", perf)


Summary: auto


**Adding performance stat applications**


Input:
*** Begin Patch
*** Update File: veena3modal/core/streaming_pipeline.py
@@
                 "audio_duration_seconds": float(audio_duration_s),
                 "ttfb_ms": float((t_first_chunk_emitted - t_start) * 1000.0 if t_first_chunk_emitted else 0.0),
                 "stream_output_kind": "delta" if _STREAM_USE_DELTA_OUTPUT else "cumulative",
                 "stream_decode_interval_tokens": int(DECODE_INTERVAL),
+                "stream_decode_interval_first_tokens": int(_STREAM_DECODE_INTERVAL_FIRST),
+                "stream_decode_interval_busy_tokens": int(_STREAM_DECODE_INTERVAL_BUSY),
+                "stream_decode_busy_pending_threshold": int(_STREAM_DECODE_BUSY_PENDING),
+                "stream_adaptive_decode": bool(_STREAM_ADAPTIVE_DECODE),
                 "stream_window_size_tokens": int(WINDOW_SIZE),
                 "stream_windowed_decode": bool(_STREAM_WINDOWED_DECODE),
             }
+            self._apply_stats(perf, "decode_interval_applied_tokens", decode_interval_applied_tokens)
+            self._apply_stats(perf, "decode_pending_requests", decode_pending_requests)
             decoder_batch_stats = {}
             if hasattr(self.snac_decoder, "get_batching_stats"):
                 try:
                     decoder_batch_stats = self.snac_decoder.get_batching_stats() or {}
*** End Patch


Output: {"output":"Success. Updated the following files:\nM veena3modal/core/streaming_pipeline.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Updating stress test for adaptive timing**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_streaming_runtime.py
@@
     timing_keys = [
         "timeline_to_first_batch_ms",
         "timeline_to_first_chunk_emitted_ms",
         "timeline_total_ms",
         "llm_time_in_queue_ms",
@@
         "bicodec_decode_wall_ms_total",
         "bicodec_decode_interval_ms_p50",
+        "decode_interval_applied_tokens_p50",
+        "decode_pending_requests_p50",
         "batch_avg_size",
         "batch_max_seen",
+        "batch_queue_wait_ms_avg",
+        "batch_compute_ms_avg",
+        "batch_queue_depth_avg",
     ]
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Blocking config save**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_streaming_runtime.py
@@
             "stream_decode_interval": args.stream_decode_interval,
             "stream_window_size": args.stream_window_size,
             "stream_min_semantic_first": args.stream_min_semantic_first,
             "stream_crossfade_ms": args.stream_crossfade_ms,
+            "adaptive_decode": args.adaptive_decode,
+            "adaptive_decode_first_interval": args.adaptive_decode_first_interval,
+            "adaptive_decode_busy_interval": args.adaptive_decode_busy_interval,
+            "adaptive_decode_busy_pending": args.adaptive_decode_busy_pending,
             "disable_windowed_decode": args.disable_windowed_decode,
             "disable_bicodec_batching": args.disable_bicodec_batching,
             "bicodec_batch_max": args.bicodec_batch_max,
             "bicodec_batch_timeout_ms": args.bicodec_batch_timeout_ms,
             "timestamp": int(time.time()),
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Adding keys to timing mapping**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_streaming_runtime.py
@@
                     "tokens_per_batch_p50": lvl.timing_stats("tokens_per_batch_p50"),
                     "bicodec_decode_calls": lvl.timing_stats("bicodec_decode_calls"),
                     "bicodec_decode_wall_ms_total": lvl.timing_stats("bicodec_decode_wall_ms_total"),
                     "bicodec_decode_interval_ms_p50": lvl.timing_stats("bicodec_decode_interval_ms_p50"),
+                    "decode_interval_applied_tokens_p50": lvl.timing_stats("decode_interval_applied_tokens_p50"),
+                    "decode_pending_requests_p50": lvl.timing_stats("decode_pending_requests_p50"),
                     "batch_avg_size": lvl.timing_stats("batch_avg_size"),
                     "batch_max_seen": lvl.timing_stats("batch_max_seen"),
+                    "batch_queue_wait_ms_avg": lvl.timing_stats("batch_queue_wait_ms_avg"),
+                    "batch_compute_ms_avg": lvl.timing_stats("batch_compute_ms_avg"),
+                    "batch_queue_depth_avg": lvl.timing_stats("batch_queue_depth_avg"),
                     "latency_over_timeline_ms": lvl.latency_over_metric_stats("timeline_total_ms"),
                 },
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Locating parser args and environment setup**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_streaming_runtime.py
@@
     parser.add_argument("--stream-decode-interval", type=int, default=48)
     parser.add_argument("--stream-window-size", type=int, default=128)
     parser.add_argument("--stream-min-semantic-first", type=int, default=10)
     parser.add_argument("--stream-crossfade-ms", type=int, default=50)
+    parser.add_argument("--adaptive-decode", action="store_true", default=False)
+    parser.add_argument("--adaptive-decode-first-interval", type=int, default=48)
+    parser.add_argument("--adaptive-decode-busy-interval", type=int, default=64)
+    parser.add_argument("--adaptive-decode-busy-pending", type=int, default=32)
     parser.add_argument("--disable-windowed-decode", action="store_true", default=False)
     parser.add_argument("--disable-bicodec-batching", action="store_true", default=False)
     parser.add_argument("--bicodec-batch-max", type=int, default=16)
     parser.add_argument("--bicodec-batch-timeout-ms", type=float, default=6.0)
@@
     os.environ["VEENA3_STREAM_DECODE_INTERVAL"] = str(args.stream_decode_interval)
     os.environ["VEENA3_STREAM_WINDOW_SIZE"] = str(args.stream_window_size)
     os.environ["VEENA3_STREAM_MIN_SEMANTIC_FIRST"] = str(args.stream_min_semantic_first)
     os.environ["VEENA3_STREAM_CROSSFADE_MS"] = str(args.stream_crossfade_ms)
+    os.environ["VEENA3_STREAM_ADAPTIVE_DECODE"] = "1" if args.adaptive_decode else "0"
+    os.environ["VEENA3_STREAM_DECODE_INTERVAL_FIRST"] = str(args.adaptive_decode_first_interval)
+    os.environ["VEENA3_STREAM_DECODE_INTERVAL_BUSY"] = str(args.adaptive_decode_busy_interval)
+    os.environ["VEENA3_STREAM_DECODE_BUSY_PENDING"] = str(args.adaptive_decode_busy_pending)
     os.environ["VEENA3_STREAM_WINDOWED_DECODE"] = "0" if args.disable_windowed_decode else "1"
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Adding compile check for config fields**


python3 -m py_compile veena3modal/core/bicodec_decoder.py veena3modal/core/streaming_pipeline.py scripts/stress_test_streaming_runtime.py


Output:
Chunk ID: acb105
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 0
Output:


**Planning adaptive decode benchmark**


Code changes compile clean. I’m now running two high-concurrency validation runs for the new adaptive decode scheduler (fast first chunk + 64-token decode stride only under queue pressure) to see if it keeps the 64-interval throughput while improving TTFB.


**Running sequential commands**


set -euo pipefail
for cfg in "adap32 32" "adap24 24"; do
  name=$(echo "$cfg" | awk '{print $1}')
  pending=$(echo "$cfg" | awk '{print $2}')
  out="/tmp/stream_cfg_${name}.json"
  log="/tmp/stream_cfg_${name}.log"
  echo "=== RUN $name ==="
  python3 scripts/stress_test_streaming_runtime.py \
    --levels 96 \
    --text short \
    --max-tokens 1024 \
    --timeout 240 \
    --requests-multiplier 1 \
    --min-requests 8 \
    --gpu-memory 0.25 \
    --stream-output-kind delta \
    --stream-decode-interval 48 \
    --stream-window-size 128 \
    --bicodec-batch-max 16 \
    --bicodec-batch-timeout-ms 6 \
    --adaptive-decode \
    --adaptive-decode-first-interval 48 \
    --adaptive-decode-busy-interval 64 \
    --adaptive-decode-busy-pending "$pending" \
    --output "$out" >"$log" 2>&1
  python3 - <<PY
import json
p='$out'
d=json.load(open(p))
r=d['levels'][0]
t=r['timing']
print(f"{p}: thr={r['throughput_rps']:.2f} lat={r['latency_ms']['avg']:.1f} ttfb={r['ttfb_ms']['avg']:.1f} llm_tok_ms={t['llm_time_per_token_ms']['avg']:.2f} bicodec_total={t['bicodec_decode_wall_ms_total']['avg']:.1f} applied_interval_p50={t.get('decode_interval_applied_tokens_p50',{}).get('avg',0):.1f} pending_p50={t.get('decode_pending_requests_p50',{}).get('avg',0):.1f} queue_wait_avg={t.get('batch_queue_wait_ms_avg',{}).get('avg',0):.1f}")
PY
done


Output:
Chunk ID: e6f56c
Wall time: 1.0013 seconds
Process running with session ID 15590
Original token count: 5
Output:
=== RUN adap32 ===


**Planning periodic updates**


Output:
Chunk ID: 37ca82
Wall time: 5.0011 seconds
Process running with session ID 15590
Original token count: 0
Output:


Output:
Chunk ID: d5622b
Wall time: 5.0008 seconds
Process running with session ID 15590
Original token count: 0
Output:


The adaptive run is still executing; after it completes I’ll compare it directly against `fixed48` and `fixed64` to verify whether we improved both throughput and first-audio latency.


Output:
Chunk ID: 446fe0
Wall time: 5.0009 seconds
Process running with session ID 15590
Original token count: 0
Output:


Output:
Chunk ID: 16470b
Wall time: 5.0016 seconds
Process running with session ID 15590
Original token count: 0
Output:


Output:
Chunk ID: 26598b
Wall time: 5.0006 seconds
Process running with session ID 15590
Original token count: 46
Output:
/tmp/stream_cfg_adap32.json: thr=23.05 lat=3406.4 ttfb=1149.4 llm_tok_ms=14.69 bicodec_total=2149.7 applied_interval_p50=62.0 pending_p50=56.7 queue_wait_avg=546.0
=== RUN adap24 ===


**Assessing throughput and overhead**


First adaptive profile (`busy_pending=32`) was not a win: throughput fell to `23.05 req/s` and queue-wait spiked, so this threshold is too aggressive. I’m finishing `busy_pending=24` now, then I’ll summarize and decide whether to keep adaptive off by default.


**Running aggressive poll test**


Output:
Chunk ID: a51899
Wall time: 5.0013 seconds
Process running with session ID 15590
Original token count: 0
Output:


Summary: auto


Output:
Chunk ID: 684807
Wall time: 5.0016 seconds
Process running with session ID 15590
Original token count: 0
Output:


Output:
Chunk ID: 45b545
Wall time: 5.0003 seconds
Process running with session ID 15590
Original token count: 0
Output:


I’m seeing the adaptive policy can accidentally increase queueing if the busy threshold is too low. I’ll confirm with this second run and then lock in the best fixed profile plus instrumentation-driven next actions.


Output:
Chunk ID: 8fe896
Wall time: 5.0009 seconds
Process running with session ID 15590
Original token count: 0
Output:


Output:
Chunk ID: b4251e
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 41
Output:
/tmp/stream_cfg_adap24.json: thr=24.45 lat=3273.4 ttfb=1113.5 llm_tok_ms=14.08 bicodec_total=1755.6 applied_interval_p50=60.0 pending_p50=47.2 queue_wait_avg=435.5


**Planning performance retesting**


I have enough to reject the first adaptive settings. I’m running two quick control runs on the current code (`fixed48` and `fixed64`) so the final comparison is apples-to-apples after instrumentation changes.


set -euo pipefail
for cfg in "fixed48 48" "fixed64 64"; do
  name=$(echo "$cfg" | awk '{print $1}')
  interval=$(echo "$cfg" | awk '{print $2}')
  out="/tmp/stream_cfg_${name}_postinst.json"
  log="/tmp/stream_cfg_${name}_postinst.log"
  echo "=== RUN $name ==="
  python3 scripts/stress_test_streaming_runtime.py \
    --levels 96 \
    --text short \
    --max-tokens 1024 \
    --timeout 240 \
    --requests-multiplier 1 \
    --min-requests 8 \
    --gpu-memory 0.25 \
    --stream-output-kind delta \
    --stream-decode-interval "$interval" \
    --stream-window-size 128 \
    --bicodec-batch-max 16 \
    --bicodec-batch-timeout-ms 6 \
    --output "$out" >"$log" 2>&1
  python3 - <<PY
import json
p='$out'
d=json.load(open(p))
r=d['levels'][0]
t=r['timing']
print(f"{p}: thr={r['throughput_rps']:.2f} lat={r['latency_ms']['avg']:.1f} ttfb={r['ttfb_ms']['avg']:.1f} llm_tok_ms={t['llm_time_per_token_ms']['avg']:.2f} bicodec_total={t['bicodec_decode_wall_ms_total']['avg']:.1f} batch_avg={t['batch_avg_size']['avg']:.2f} queue_wait_avg={t.get('batch_queue_wait_ms_avg',{}).get('avg',0):.1f}")
PY
done


Output:
Chunk ID: 17cad7
Wall time: 1.0008 seconds
Process running with session ID 75511
Original token count: 5
Output:
=== RUN fixed48 ===


**Planning polling updates**


Output:
Chunk ID: 7498f9
Wall time: 5.0013 seconds
Process running with session ID 75511
Original token count: 0
Output:


Output:
Chunk ID: 0597f1
Wall time: 5.0011 seconds
Process running with session ID 75511
Original token count: 0
Output:


Control runs are in progress; I’m using these to isolate whether any metric movement is from the new instrumentation itself versus scheduler settings.


Output:
Chunk ID: 7fb0c6
Wall time: 5.0006 seconds
Process running with session ID 75511
Original token count: 0
Output:


Output:
Chunk ID: 749c92
Wall time: 5.0004 seconds
Process running with session ID 75511
Original token count: 0
Output:


Output:
Chunk ID: ab4308
Wall time: 5.0007 seconds
Process running with session ID 75511
Original token count: 42
Output:
/tmp/stream_cfg_fixed48_postinst.json: thr=21.68 lat=3644.3 ttfb=1047.8 llm_tok_ms=15.70 bicodec_total=2541.7 batch_avg=10.21 queue_wait_avg=501.8
=== RUN fixed64 ===


**Analyzing throughput variability causes**


First control (`fixed48`) came in lower than earlier results, so I’m waiting on `fixed64` to determine whether this is a transient runtime condition or a real regression from the latest changes.


Output:
Chunk ID: ba5d50
Wall time: 5.0009 seconds
Process running with session ID 75511
Original token count: 0
Output:


Output:
Chunk ID: 2f26db
Wall time: 5.0003 seconds
Process running with session ID 75511
Original token count: 0
Output:


Output:
Chunk ID: 98fd4b
Wall time: 5.0008 seconds
Process running with session ID 75511
Original token count: 0
Output:


`fixed64` is still running; if it also drops materially, I’ll bisect the just-added logic to remove any scheduling overhead and keep only low-cost instrumentation.


Output:
Chunk ID: 065256
Wall time: 3.4606 seconds
Process exited with code 0
Original token count: 37
Output:
/tmp/stream_cfg_fixed64_postinst.json: thr=22.92 lat=3370.2 ttfb=1208.9 llm_tok_ms=14.24 bicodec_total=2004.7 batch_avg=9.87 queue_wait_avg=526.0


**Planning deterministic benchmarking patch**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_streaming_runtime.py
@@
 async def run_one_request(
     text: str,
     speaker: str,
     max_tokens: int,
     timeout_s: float,
     enable_chunking: bool,
+    seed: Optional[int],
 ) -> RequestResult:
@@
         gen = tts_runtime.generate_speech_streaming(
             text=text,
             speaker=speaker,
             max_tokens=max_tokens,
             enable_chunking=enable_chunking,
+            seed=seed,
         )
@@
 async def run_level(
     concurrency: int,
     total_requests: int,
     text: str,
     max_tokens: int,
     timeout_s: float,
     enable_chunking: bool,
     gpu_monitor: GPUMonitor,
+    seed_base: Optional[int],
 ) -> LevelResult:
@@
     async def wrapped(i: int) -> RequestResult:
         async with semaphore:
             speaker = SPEAKERS[i % len(SPEAKERS)]
+            seed = (seed_base + i) if seed_base is not None else None
             return await run_one_request(
                 text=text,
                 speaker=speaker,
                 max_tokens=max_tokens,
                 timeout_s=timeout_s,
                 enable_chunking=enable_chunking,
+                seed=seed,
             )
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Adding initialization calls in main_async**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_streaming_runtime.py
@@
     await run_one_request(
         text=text,
         speaker=SPEAKERS[0],
         max_tokens=args.max_tokens,
         timeout_s=args.timeout,
         enable_chunking=args.chunking,
+        seed=args.seed_base,
     )
@@
             timeout_s=args.timeout,
             enable_chunking=args.chunking,
             gpu_monitor=gpu_monitor,
+            seed_base=args.seed_base,
         )
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Extending parser config with new parameters**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_streaming_runtime.py
@@
             "max_tokens": args.max_tokens,
             "timeout_s": args.timeout,
+            "seed_base": args.seed_base,
             "requests_multiplier": args.requests_multiplier,
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Adding parser argument near token limit**


Input:
*** Begin Patch
*** Update File: scripts/stress_test_streaming_runtime.py
@@
     parser.add_argument("--text", choices=["short", "medium", "long"], default="short")
     parser.add_argument("--max-tokens", type=int, default=512, help="max_tokens sent to streaming generation")
+    parser.add_argument("--seed-base", type=int, default=1337, help="Base random seed; request i uses seed_base+i")
     parser.add_argument("--timeout", type=float, default=180.0, help="Per-request timeout seconds")
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_streaming_runtime.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Preparing compile check updates**


python3 -m py_compile scripts/stress_test_streaming_runtime.py veena3modal/core/streaming_pipeline.py veena3modal/core/bicodec_decoder.py


Output:
Chunk ID: 359018
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 0
Output:


**Running deterministic comparisons**


I added deterministic seeding to the stress harness (`--seed-base`) so comparisons are stable across runs. I’m now rerunning `fixed48`, `fixed64`, and one adaptive profile with the same seeds for an accurate ranking.


**Running sequential tests with seed**


set -euo pipefail
for cfg in "seed_fixed48 48 0" "seed_fixed64 64 0" "seed_adap24 48 1"; do
  name=$(echo "$cfg" | awk '{print $1}')
  interval=$(echo "$cfg" | awk '{print $2}')
  adaptive=$(echo "$cfg" | awk '{print $3}')
  out="/tmp/stream_cfg_${name}.json"
  log="/tmp/stream_cfg_${name}.log"
  echo "=== RUN $name ==="
  cmd=(python3 scripts/stress_test_streaming_runtime.py
    --levels 96
    --text short
    --max-tokens 1024
    --seed-base 2026
    --timeout 240
    --requests-multiplier 1
    --min-requests 8
    --gpu-memory 0.25
    --stream-output-kind delta
    --stream-decode-interval "$interval"
    --stream-window-size 128
    --bicodec-batch-max 16
    --bicodec-batch-timeout-ms 6
    --output "$out")
  if [ "$adaptive" = "1" ]; then
    cmd+=(--adaptive-decode --adaptive-decode-first-interval 48 --adaptive-decode-busy-interval 64 --adaptive-decode-busy-pending 24)
  fi
  "${cmd[@]}" >"$log" 2>&1
  python3 - <<PY
import json
p='$out'
d=json.load(open(p))
r=d['levels'][0]
t=r['timing']
print(f"{p}: thr={r['throughput_rps']:.2f} lat={r['latency_ms']['avg']:.1f} ttfb={r['ttfb_ms']['avg']:.1f} llm_tok_ms={t['llm_time_per_token_ms']['avg']:.2f} bicodec_total={t['bicodec_decode_wall_ms_total']['avg']:.1f} interval_p50={t.get('decode_interval_applied_tokens_p50',{}).get('avg',0):.1f} queue_wait_avg={t.get('batch_queue_wait_ms_avg',{}).get('avg',0):.1f}")
PY
done


Output:
Chunk ID: e8a162
Wall time: 1.0001 seconds
Process running with session ID 93389
Original token count: 7
Output:
=== RUN seed_fixed48 ===


Summary: auto


Output:
Chunk ID: f79b65
Wall time: 5.0005 seconds
Process running with session ID 93389
Original token count: 0
Output:


Output:
Chunk ID: e7a015
Wall time: 5.0008 seconds
Process running with session ID 93389
Original token count: 0
Output:


Deterministic reruns are underway with fixed seeds; this will give us a clean comparison without request-length randomness skewing throughput.


Output:
Chunk ID: 3f57fa
Wall time: 5.0003 seconds
Process running with session ID 93389
Original token count: 0
Output:


Output:
Chunk ID: 49d13e
Wall time: 5.0008 seconds
Process running with session ID 93389
Original token count: 0
Output:


Output:
Chunk ID: e3ffb5
Wall time: 5.0012 seconds
Process running with session ID 93389
Original token count: 43
Output:
/tmp/stream_cfg_seed_fixed48.json: thr=23.90 lat=3524.3 ttfb=1123.7 llm_tok_ms=15.13 bicodec_total=2250.8 interval_p50=48.0 queue_wait_avg=461.8
=== RUN seed_fixed64 ===


Seeded `fixed48` completed at `23.90 req/s`. I’m now on seeded `fixed64`; this should tell us if larger decode stride still wins when run conditions are controlled.


Output:
Chunk ID: 5a487f
Wall time: 5.0017 seconds
Process running with session ID 93389
Original token count: 0
Output:


Output:
Chunk ID: f69581
Wall time: 5.0033 seconds
Process running with session ID 93389
Original token count: 0
Output:


Output:
Chunk ID: d5883a
Wall time: 5.0009 seconds
Process running with session ID 93389
Original token count: 0
Output:


**Adjusting update interval**


Still running `fixed64`; once this and the seeded adaptive run finish, I’ll lock in the best scheduler profile and provide concrete next optimization steps.


Output:
Chunk ID: 11b47e
Wall time: 5.0012 seconds
Process running with session ID 93389
Original token count: 43
Output:
/tmp/stream_cfg_seed_fixed64.json: thr=22.47 lat=3464.4 ttfb=1280.2 llm_tok_ms=14.78 bicodec_total=1975.9 interval_p50=64.0 queue_wait_avg=513.0
=== RUN seed_adap24 ===


**Noticing fixed64 lower throughput**


Seeded `fixed64` did not beat `fixed48` in this run and had worse TTFB, so we need to treat 64 as load-sensitive rather than universally better. I’m finishing seeded adaptive now for final ranking.


**Polling for update**


Output:
Chunk ID: 0774df
Wall time: 5.0035 seconds
Process running with session ID 93389
Original token count: 0
Output:


Output:
Chunk ID: fc5201
Wall time: 5.0011 seconds
Process running with session ID 93389
Original token count: 0
Output:


Output:
Chunk ID: 2e6b11
Wall time: 5.0021 seconds
Process running with session ID 93389
Original token count: 0
Output:


Adaptive seeded run is still processing. After this, I’ll aggregate all benchmark JSONs into one comparative table and call out bottleneck signatures from the new queue-wait metrics.


Output:
Chunk ID: 96b63f
Wall time: 5.0016 seconds
Process running with session ID 93389
Original token count: 0
Output:


Output:
Chunk ID: 415a09
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 36
Output:
/tmp/stream_cfg_seed_adap24.json: thr=21.94 lat=3562.2 ttfb=1173.1 llm_tok_ms=15.12 bicodec_total=1904.7 interval_p50=58.8 queue_wait_avg=420.8


**Planning run results aggregation**


python3 - <<'PY'
import json,glob,os,re
rows=[]
for p in sorted(glob.glob('/tmp/stream_cfg_*.json')):
    try:
        d=json.load(open(p))
    except Exception:
        continue
    levels=d.get('levels',[])
    if not levels:
        continue
    r=levels[0]
    cfg=d.get('config',{})
    t=r.get('timing',{})
    rows.append({
        'file':os.path.basename(p),
        'thr':r.get('throughput_rps',0),
        'lat':r.get('latency_ms',{}).get('avg',0),
        'ttfb':r.get('ttfb_ms',{}).get('avg',0),
        'llm_tok_ms':t.get('llm_time_per_token_ms',{}).get('avg',0),
        'bicodec_ms':t.get('bicodec_decode_wall_ms_total',{}).get('avg',0),
        'batch_avg':t.get('batch_avg_size',{}).get('avg',0),
        'queue_wait':t.get('batch_queue_wait_ms_avg',{}).get('avg',0),
        'interval_p50':t.get('decode_interval_applied_tokens_p50',{}).get('avg',0),
        'adaptive':cfg.get('adaptive_decode',False),
        'interval':cfg.get('stream_decode_interval'),
        'window':cfg.get('stream_window_size'),
        'bmax':cfg.get('bicodec_batch_max'),
        'btimeout':cfg.get('bicodec_batch_timeout_ms'),
        'seed_base':cfg.get('seed_base'),
    })
rows=sorted(rows,key=lambda x:x['thr'],reverse=True)
print('count',len(rows))
for r in rows:
    print(f"{r['file']}: thr={r['thr']:.2f} lat={r['lat']:.0f} ttfb={r['ttfb']:.0f} llm_tok_ms={r['llm_tok_ms']:.2f} bicodec={r['bicodec_ms']:.0f} batch_avg={r['batch_avg']:.2f} qwait={r['queue_wait']:.0f} interval={r['interval']} interval_p50={r['interval_p50']:.1f} window={r['window']} bmax={r['bmax']} bto={r['btimeout']} adaptive={r['adaptive']} seed={r['seed_base']}")
PY


Output:
Chunk ID: 85abf7
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 638
Output:
count 13
stream_cfg_i64_t6_b16_w128.json: thr=26.19 lat=3205 ttfb=1196 llm_tok_ms=13.71 bicodec=1601 batch_avg=10.67 qwait=0 interval=64 interval_p50=0.0 window=128 bmax=16 bto=6.0 adaptive=False seed=None
stream_cfg_adap24.json: thr=24.45 lat=3273 ttfb=1114 llm_tok_ms=14.08 bicodec=1756 batch_avg=10.66 qwait=436 interval=48 interval_p50=60.0 window=128 bmax=16 bto=6.0 adaptive=True seed=None
stream_cfg_seed_fixed48.json: thr=23.90 lat=3524 ttfb=1124 llm_tok_ms=15.13 bicodec=2251 batch_avg=10.20 qwait=462 interval=48 interval_p50=48.0 window=128 bmax=16 bto=6.0 adaptive=False seed=2026
stream_cfg_i48_t6_b16_w96.json: thr=23.21 lat=3510 ttfb=1071 llm_tok_ms=15.16 bicodec=2592 batch_avg=9.86 qwait=0 interval=48 interval_p50=0.0 window=96 bmax=16 bto=6.0 adaptive=False seed=None
stream_cfg_i48_t8_b24_w128.json: thr=23.12 lat=3544 ttfb=966 llm_tok_ms=15.14 bicodec=2064 batch_avg=15.87 qwait=0 interval=48 interval_p50=0.0 window=128 bmax=24 bto=8.0 adaptive=False seed=None
stream_cfg_i32_t6_b16_w128.json: thr=23.09 lat=3723 ttfb=922 llm_tok_ms=16.23 bicodec=3150 batch_avg=11.40 qwait=0 interval=32 interval_p50=0.0 window=128 bmax=16 bto=6.0 adaptive=False seed=None
stream_cfg_adap32.json: thr=23.05 lat=3406 ttfb=1149 llm_tok_ms=14.69 bicodec=2150 batch_avg=8.12 qwait=546 interval=48 interval_p50=62.0 window=128 bmax=16 bto=6.0 adaptive=True seed=None
stream_cfg_fixed64_postinst.json: thr=22.92 lat=3370 ttfb=1209 llm_tok_ms=14.24 bicodec=2005 batch_avg=9.87 qwait=526 interval=64 interval_p50=64.0 window=128 bmax=16 bto=6.0 adaptive=False seed=None
stream_cfg_seed_fixed64.json: thr=22.47 lat=3464 ttfb=1280 llm_tok_ms=14.78 bicodec=1976 batch_avg=7.96 qwait=513 interval=64 interval_p50=64.0 window=128 bmax=16 bto=6.0 adaptive=False seed=2026
stream_cfg_seed_adap24.json: thr=21.94 lat=3562 ttfb=1173 llm_tok_ms=15.12 bicodec=1905 batch_avg=8.44 qwait=421 interval=48 interval_p50=58.8 window=128 bmax=16 bto=6.0 adaptive=True seed=2026
stream_cfg_fixed48_postinst.json: thr=21.68 lat=3644 ttfb=1048 llm_tok_ms=15.70 bicodec=2542 batch_avg=10.21 qwait=502 interval=48 interval_p50=48.0 window=128 bmax=16 bto=6.0 adaptive=False seed=None
stream_cfg_i48_t8_b16_w128.json: thr=21.65 lat=3610 ttfb=998 llm_tok_ms=15.51 bicodec=2228 batch_avg=10.60 qwait=0 interval=48 interval_p50=0.0 window=128 bmax=16 bto=8.0 adaptive=False seed=None
stream_cfg_i48_t4_b16_w128.json: thr=21.44 lat=3676 ttfb=1071 llm_tok_ms=16.00 bicodec=2907 batch_avg=7.44 qwait=0 interval=48 interval_p50=0.0 window=128 bmax=16 bto=4.0 adaptive=False seed=None


**Planning baseline comparison data**


python3 - <<'PY'
import json
for p in ['/tmp/stream_baseline_levels_detailed.json','/tmp/stream_levels_with_bicodec_batch.json']:
    d=json.load(open(p))
    levels=d.get('levels',[])
    by={r['concurrency']:r for r in levels}
    r=by.get(96)
    t=r['timing']
    print(p)
    print(f"thr={r['throughput_rps']:.2f} lat={r['latency_ms']['avg']:.1f} ttfb={r['ttfb_ms']['avg']:.1f} llm_tok_ms={t['llm_time_per_token_ms']['avg']:.2f} bicodec={t['bicodec_decode_wall_ms_total']['avg']:.1f} batch_avg={t.get('batch_avg_size',{}).get('avg',0):.2f}")
PY


Output:
Chunk ID: 0b5c1c
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 61
Output:
/tmp/stream_baseline_levels_detailed.json
thr=18.92 lat=5069.5 ttfb=1979.0 llm_tok_ms=22.09 bicodec=4293.1 batch_avg=0.00
/tmp/stream_levels_with_bicodec_batch.json
thr=24.26 lat=3225.0 ttfb=927.7 llm_tok_ms=13.96 bicodec=1905.3 batch_avg=9.59


**Planning benchmark runs**


I’ve consolidated the sweep data and now I’m running one deterministic full concurrency ladder (`32,64,96`) on the current optimized path so you get a clean, final stress profile with the new queue-wait instrumentation.


**Running seeded command**


python3 scripts/stress_test_streaming_runtime.py --levels 32,64,96 --text short --max-tokens 1024 --seed-base 2026 --timeout 240 --requests-multiplier 1 --min-requests 8 --gpu-memory 0.25 --stream-output-kind delta --stream-decode-interval 48 --stream-window-size 128 --bicodec-batch-max 16 --bicodec-batch-timeout-ms 6 --output /tmp/stream_final_ladder_seeded.json


Output:
Chunk ID: 2671dd
Wall time: 1.0004 seconds
Process running with session ID 41173
Original token count: 0
Output:


Summary: auto


**Polling for script output**


Output:
Chunk ID: 4aea20
Wall time: 5.0003 seconds
Process running with session ID 41173
Original token count: 1585
Output:
[2026-02-18 18:38:33] INFO tts_runtime.py:183: Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.25
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-18 18:38:34 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-18 18:38:34 [model.py:1561] Using max model len 4096
INFO 02-18 18:38:34 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-18 18:38:34 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:35 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:37 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:57053 backend=nccl
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:37 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:38 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:38 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=95252) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=95252) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=95252)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]


**Managing verbose model output**


The full ladder run is active and currently in model-load startup. I’ll keep this run going and then extract only compact per-level metrics from the result JSON to avoid noisy logs.


**Adjusting token limit for polling**


Output:
Chunk ID: 0178b4
Wall time: 5.0018 seconds
Process running with session ID 41173
Original token count: 403
Output:
(EngineCore_DP0 pid=95252)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.11it/s]
(EngineCore_DP0 pid=95252)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.11it/s]
(EngineCore_DP0 pid=95252)
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:39 [default_loader.py:291] Loading weights took 0.98 seconds
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:40 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.514071 seconds
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:45 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:45 [backends.py:872] Dynamo bytecode transform time: 4.19 s
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:47 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.631 s
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:47 [monitor.py:34] torch.compile takes 4.83 s in total
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:48 [gpu_worker.py:356] Available KV cache memory: 18.06 GiB
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:48 [kv_cache_utils.py:1307] GPU KV cache size: 1,577,712 tokens
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:48 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 385.18x
(EngineCore_DP0 pid=95252)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 33.23it/s]


**Continuing graph capture**


Output:
Chunk ID: 8b6e19
Wall time: 5.0010 seconds
Process running with session ID 41173
Original token count: 575
Output:

Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 34.23it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 35.67it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 36.36it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 36.66it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 36.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 36.78it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 35.43it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:00<00:00, 35.13it/s]
(EngineCore_DP0 pid=95252)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 29.23it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 33.67it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 34.96it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 35.36it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.88it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.07it/s]
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:51 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=95252) INFO 02-18 18:38:51 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.54 seconds
[2026-02-18 18:38:52] INFO tts_runtime.py:192: Initializing prompt builder...
[2026-02-18 18:38:52] INFO tts_runtime.py:199: Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


**Preparing benchmark output**


Output:
Chunk ID: 7a7e16
Wall time: 5.0010 seconds
Process running with session ID 41173
Original token count: 127
Output:
[2026-02-18 18:39:00] INFO tts_runtime.py:209: Initializing TTS pipeline...
[2026-02-18 18:39:00] INFO tts_runtime.py:220: Initializing streaming pipeline...
[2026-02-18 18:39:00] INFO long_text_processor.py:112: LongTextProcessor initialized
[2026-02-18 18:39:00] INFO tts_runtime.py:277: ✅ TTS runtime initialized in 31969ms
[2026-02-18 18:39:00] INFO tts_runtime.py:278:    Model version: LLM
[2026-02-18 18:39:00] INFO tts_runtime.py:289: Speaker globals cache startup precompute skipped (disabled)


**Considering poll for outputs**


Initialization just completed; I’m waiting for the three concurrency levels to finish and then I’ll extract the concise ladder metrics from `/tmp/stream_final_ladder_seeded.json`.


**Adding short polling delay**


Output:
Chunk ID: dd840e
Wall time: 4.8701 seconds
Process exited with code 0
Original token count: 1664
Output:
Total output lines: 117

✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cuda
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
   ℹ  Micro-batching enabled (max_batch=16, timeout_ms=6.0)
🚀 SparkTTSPipeline initialized
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)

=== streaming concurrency=32 requests=32 ===
success=32/32 (100%)
wall=2.40s throughput=13.36 req/s
latency_ms avg=1957 p50=1911 p95=2314 max=2394
ttfb_ms avg=680 p50=655 p95=705 max=705
chunks_per_req avg=5.5 p50=5.0 p95=6.0 max=6.0
inter_chunk_ms avg=275.4 p50=226.2 p95=914.7 max=1035.7
tail_emit_speed_x avg=3.14 p50=3.14 p95=4.27 max=4.80
timeline_to_first_batch_ms: avg=46.12 p50=45.70 p95=51.22 max=52.26
timeline_to_first_chunk_emitted_ms: avg=672.01 p50=653.61 p95=695.63 max=696.23
timeline_total_ms: avg=1911.26 p50=1909.18 p95=2302.33 max=2380.70
llm_time_in_queue_ms: avg=0.07 p50=0.07 p95=0.15 max=0.17
llm_scheduler_ms: avg=36.32 p50=36.90 p95=36.90 max=39.24
llm_first_token_ms: avg=44.94 p50=44.47 p95=50.30 max=51.39
llm_request_lifecycle_ms: avg=1494.54 p50=1497.78 p95=1620.28 max=1629.10
llm_generation_wall_ms: avg=1885.90 p50=1877.01 p95=2269.80 max=2348.67
llm_time_per_token_ms: avg=8.25 p50=8.06 p95=9.64 max=9.82
llm_batch_wall_ms_total: avg=1885.85 p50=1876.91 p95=2269.72 max=2348.59
llm_decode_wall_ms_total: avg=1839.76 p50=1824.69 p95=2227.30 max=2307.69
tokens_per_batch_p50: avg=1.00 p50=1.00 p95=1.00 max=1.00
bicodec_decode_calls: avg=4.50 p50=4.00 p95=5.00 max=5.00
bicodec_decode_wall_ms_total: avg=819.71 p50=778.87 p95=1197.62 max=1266.05
bicodec_decode_interval_ms_p50: avg=270.65 p50=226.25 p95=394.48 max=394.60
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
batch_avg_size: avg=10.18 p50=10.20 p95=12.25 max=12.25
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_queue_wait_ms_avg: avg=83.26 p50=68.30 p95=120.11 max=127.55
batch_compute_ms_avg: avg=175.39 p50=178.34 p95=213.29 max=213.29
batch_queue_depth_avg: avg=6.72 p50=6.78 p95=8.08 max=8.08
latency_over_timeline_ms: avg=46.11 p50=22.67 p95=160.11 max=228.70
timeline_sh…464 tokens truncated…eue_depth_avg: avg=16.20 p50=16.20 p95=16.52 max=16.70
latency_over_timeline_ms: avg=62.91 p50=26.33 p95=195.09 max=259.48
timeline_share: llm=99.5% bicodec=42.2% other=0.0%
gpu: util_avg=61% util_max=96% mem_avg=27262MB mem_max=27635MB

=== streaming concurrency=96 requests=96 ===
success=96/96 (100%)
wall=3.99s throughput=24.07 req/s
latency_ms avg=3378 p50=3459 p95=3986 max=3987
ttfb_ms avg=984 p50=954 p95=1183 max=1183
chunks_per_req avg=5.1 p50=5.0 p95=6.0 max=6.0
inter_chunk_ms avg=572.0 p50=605.2 p95=1492.8 max=1619.3
tail_emit_speed_x avg=1.66 p50=1.52 p95=2.22 max=2.39
timeline_to_first_batch_ms: avg=61.76 p50=62.47 p95=70.39 max=71.64
timeline_to_first_chunk_emitted_ms: avg=963.49 p50=937.61 p95=1147.77 max=1149.11
timeline_total_ms: avg=3334.87 p50=3441.63 p95=3951.06 max=3953.67
llm_time_in_queue_ms: avg=0.13 p50=0.12 p95=0.28 max=0.31
llm_scheduler_ms: avg=43.38 p50=36.94 p95=55.01 max=55.01
llm_first_token_ms: avg=58.89 p50=59.76 p95=68.54 max=70.65
llm_request_lifecycle_ms: avg=2429.79 p50=2451.75 p95=2667.09 max=2750.71
llm_generation_wall_ms: avg=3322.83 p50=3441.24 p95=3951.02 max=3953.63
llm_time_per_token_ms: avg=14.51 p50=14.81 p95=16.88 max=17.19
llm_batch_wall_ms_total: avg=3020.01 p50=3021.94 p95=3671.10 max=3790.86
llm_decode_wall_ms_total: avg=2958.29 p50=2961.18 p95=3605.84 max=3730.82
tokens_per_batch_p50: avg=1.00 p50=1.00 p95=1.00 max=1.00
bicodec_decode_calls: avg=4.15 p50=4.00 p95=5.00 max=5.00
bicodec_decode_wall_ms_total: avg=1903.79 p50=1839.93 p95=3123.45 max=3125.28
bicodec_decode_interval_ms_p50: avg=673.73 p50=655.43 p95=929.59 max=1085.92
decode_interval_applied_tokens_p50: avg=48.00 p50=48.00 p95=48.00 max=48.00
decode_pending_requests_p50: avg=0.00 p50=0.00 p95=0.00 max=0.00
batch_avg_size: avg=9.44 p50=9.46 p95=10.33 max=10.48
batch_max_seen: avg=16.00 p50=16.00 p95=16.00 max=16.00
batch_queue_wait_ms_avg: avg=267.78 p50=287.62 p95=321.65 max=321.65
batch_compute_ms_avg: avg=81.11 p50=80.94 p95=89.90 max=91.17
batch_queue_depth_avg: avg=29.16 p50=30.67 p95=31.28 max=31.30
latency_over_timeline_ms: avg=43.04 p50=33.35 p95=109.75 max=158.26
timeline_share: llm=99.6% bicodec=57.1% other=0.0%
gpu: util_avg=79% util_max=96% mem_avg=27635MB mem_max=27635MB

results saved: /tmp/stream_final_ladder_seeded.json
ERROR 02-18 18:39:15 [core_client.py:605] Engine core proc EngineCore_DP0 died unexpectedly, shutting down client.


**Planning ladder file parsing**


python3 - <<'PY'
import json
p='/tmp/stream_final_ladder_seeded.json'
d=json.load(open(p))
for r in d['levels']:
    c=r['concurrency']
    t=r['timing']
    print(f"c={c} thr={r['throughput_rps']:.2f} lat={r['latency_ms']['avg']:.1f} ttfb={r['ttfb_ms']['avg']:.1f} llm_tok_ms={t['llm_time_per_token_ms']['avg']:.2f} bicodec={t['bicodec_decode_wall_ms_total']['avg']:.1f} batch_avg={t['batch_avg_size']['avg']:.2f} qwait={t['batch_queue_wait_ms_avg']['avg']:.1f} qdepth={t['batch_queue_depth_avg']['avg']:.1f} util={r['gpu']['gpu_util_pct']['avg']:.1f}")
PY


Output:
Chunk ID: 707362
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 88
Output:
c=32 thr=13.36 lat=1957.4 ttfb=679.7 llm_tok_ms=8.25 bicodec=819.7 batch_avg=10.18 qwait=83.3 qdepth=6.7 util=55.2
c=64 thr=21.67 lat=2666.2 ttfb=819.0 llm_tok_ms=11.45 bicodec=1098.8 batch_avg=9.44 qwait=170.9 qdepth=16.2 util=61.2
c=96 thr=24.07 lat=3377.9 ttfb=984.0 llm_tok_ms=14.51 bicodec=1903.8 batch_avg=9.44 qwait=267.8 qdepth=29.2 util=78.7


**Identifying batch queue bottleneck**


git diff -- veena3modal/core/bicodec_decoder.py veena3modal/core/streaming_pipeline.py scripts/stress_test_streaming_runtime.py | sed -n '1,260p'


Output:
Chunk ID: 1a7c47
Wall time: 0.0506 seconds
Process exited with code 0
Original token count: 183
Output:
 D CLEANUP_PLAN.md
 D PROGRESS.md
 D migration.md
 M veena3modal/api/fastapi_app.py
 M veena3modal/core/bicodec_decoder.py
 M veena3modal/core/constants.py
 M veena3modal/core/pipeline.py
 M veena3modal/core/streaming_pipeline.py
 M veena3modal/services/tts_runtime.py
?? scripts/profile_pipeline.py
?? scripts/setup_local.sh
?? scripts/stress_test_local.py
?? scripts/stress_test_optimized_local.py
?? scripts/stress_test_production.py
?? scripts/stress_test_runtime_detailed.py
?? scripts/stress_test_streaming_runtime.py
?? stress_results_opt_local.json
?? veena3modal/__main__.py
?? veena3modal/core/bicodec_batch_decoder.py
?? veena3modal/core/dual_engine.py
?? veena3modal/core/multi_engine.py
?? veena3modal/local_server.py


Output:
Chunk ID: ca23d9
Wall time: 0.0518 seconds
Process exited with code 0
Original token count: 2560
Output:
diff --git a/veena3modal/core/bicodec_decoder.py b/veena3modal/core/bicodec_decoder.py
index 820609a..2c5fa1c 100644
--- a/veena3modal/core/bicodec_decoder.py
+++ b/veena3modal/core/bicodec_decoder.py
@@ -5,11 +5,18 @@ Decodes BiCodec audio tokens (semantic + global) to audio waveforms.
 Replaces SNAC decoder for Spark TTS model.
 """

-import torch
-import numpy as np
+import asyncio
+import logging
+import os
 import sys
+import threading
+import time
+from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+import torch

 # Ensure `sparktts` (vendored under external/) is importable in local/dev runs.
 # In Modal, PYTHONPATH already includes `/root/external/sparktts`.
@@ -32,6 +39,47 @@ from veena3modal.core.constants import (
     AUDIO_SAMPLE_RATE,
 )

+logger = logging.getLogger(__name__)
+
+
+def _env_flag(name: str, default: bool) -> bool:
+    raw = os.environ.get(name, "")
+    if not raw:
+        return default
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int, minimum: int = 1) -> int:
+    raw = os.environ.get(name, "")
+    if not raw:
+        return default
+    try:
+        value = int(raw)
+    except ValueError:
+        return default
+    return max(minimum, value)
+
+
+def _env_float(name: str, default: float, minimum: float = 0.0) -> float:
+    raw = os.environ.get(name, "")
+    if not raw:
+        return default
+    try:
+        value = float(raw)
+    except ValueError:
+        return default
+    return max(minimum, value)
+
+
+@dataclass
+class _BatchDecodeRequest:
+    semantic_ids: List[int]
+    global_ids: List[int]
+    use_sliding_window: bool
+    trim_warmup: bool
+    future: asyncio.Future
+    submitted_at: float
+

 class BiCodecDecoder:
     """
@@ -59,7 +107,31 @@ class BiCodecDecoder:
         """
         self.device = torch.device(device if torch.cuda.is_available() else "cpu")
         self.model_path = model_path
-        self.enable_batching = enable_batching  # For streaming pipeline compatibility
+        requested_batching = bool(enable_batching)
+        if _env_flag("VEENA3_BICODEC_BATCHING_FORCE_ON", False):
+            requested_batching = True
+        if _env_flag("VEENA3_BICODEC_BATCHING_FORCE_OFF", False):
+            requested_batching = False
+        self.enable_batching = requested_batching and self.device.type == "cuda"
+        self.batch_max_size = _env_int("VEENA3_BICODEC_BATCH_MAX", 16, minimum=1)
+        self.batch_timeout_s = _env_float("VEENA3_BICODEC_BATCH_TIMEOUT_MS", 6.0, minimum=0.1) / 1000.0
+
+        # Async micro-batcher state (per event loop).
+        self._batch_map_lock = threading.Lock()
+        self._batch_queues: Dict[int, asyncio.Queue] = {}
+        self._batch_workers: Dict[int, asyncio.Task] = {}
+        self._batch_total_batches = 0
+        self._batch_total_requests = 0
+        self._batch_max_seen = 0
+        self._batch_queue_wait_ms_total = 0.0
+        self._batch_queue_wait_ms_min: Optional[float] = None
+        self._batch_queue_wait_ms_max = 0.0
+        self._batch_compute_ms_total = 0.0
+        self._batch_compute_ms_min: Optional[float] = None
+        self._batch_compute_ms_max = 0.0
+        self._batch_queue_depth_total = 0
+        self._batch_queue_depth_max = 0
+        self._batch_inflight_requests = 0

         print(f"🎵 Loading BiCodec audio tokenizer from {model_path}...")
         print(f"   Device: {self.device}")
@@ -67,9 +139,22 @@ class BiCodecDecoder:
         # Initialize BiCodec tokenizer
         self.audio_tokenizer = BiCodecTokenizer(model_path, device=str(self.device))

+        # OPTIMIZATION: Ensure model is on device once at init, not per-decode call
+        self.audio_tokenizer.device = self.device
+        self.audio_tokenizer.model.to(self.device)
+
+        # NOTE: torch.compile disabled for BiCodec -- dynamic semantic sequence lengths
+        # cause repeated recompilation (~120s each), worse than eager mode.
+        # torch.compile works best with static shapes. If needed, pre-warm with
+        # representative lengths or use mode="default" instead of "reduce-overhead".
+        # The multi-engine approach (Tier 3) provides more impactful gains.
+
         print(f"✅ BiCodec decoder initialized (sample rate: {AUDIO_SAMPLE_RATE}Hz)")
-        if enable_batching:
-            print(f"   ℹ  Batching enabled (for streaming support)")
+        if self.enable_batching:
+            print(
+                "   ℹ  Micro-batching enabled "
+                f"(max_batch={self.batch_max_size}, timeout_ms={self.batch_timeout_s * 1000:.1f})"
+            )

     def decode(
         self,
@@ -115,10 +200,7 @@ class BiCodecDecoder:
             pred_semantic = torch.tensor(semantic_ids).long().unsqueeze(0).to(self.device)
             pred_global = torch.tensor(global_ids).long().unsqueeze(0).to(self.device)

-            # Ensure audio tokenizer is on correct device
-            self.audio_tokenizer.device = self.device
-            self.audio_tokenizer.model.to(self.device)
-
+            # Device migration done once in __init__, not per-call
             # Decode through BiCodec
             # BiCodecTokenizer.detokenize() expects:
             #   - global_tokens: (batch, global_dim) -> will unsqueeze(1) internally
@@ -163,6 +245,225 @@ class BiCodecDecoder:
         audio_int16 = (audio * 32767).astype(np.int16)

         return audio_int16.tobytes()
+
+    async def decode_to_bytes_async(
+        self,
+        semantic_ids: List[int],
+        global_ids: List[int],
+    ) -> Optional[bytes]:
+        """
+        Async wrapper for decode_to_bytes.
+
+        Runs decode in the default executor so the event loop stays responsive
+        under high request concurrency.
+        """
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(
+            None,
+            self.decode_to_bytes,
+            semantic_ids,
+            global_ids,
+        )
+
+    @staticmethod
+    def _normalize_global_ids(global_ids: List[int]) -> List[int]:
+        expected = 32
+        if len(global_ids) >= expected:
+            return list(global_ids[:expected])
+        return list(global_ids) + [0] * (expected - len(global_ids))
+
+    def _decode_streaming_batch_sync(
+        self,
+        batch_items: List[Tuple[List[int], List[int], bool, bool]],
+    ) -> List[Optional[bytes]]:
+        """
+        Run a single batched BiCodec forward pass for multiple stream decode requests.
+
+        Falls back to sequential decode_streaming if shapes or settings are not batch-safe.
+        """
+        if not batch_items:
+            return []
+        if len(batch_items) == 1:
+            sem, glob, use_sw, trim = batch_items[0]
+            return [self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)]
+
+        # Sliding window path uses special trimming behavior; keep it sequential.
+        if any(use_sw for _, _, use_sw, _ in batch_items):
+            return [
+                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
+                for sem, glob, use_sw, trim in batch_items
+            ]
+
+        # Decoder needs at least 8 semantic tokens. Use fallback if any request is below.
+        if any(len(sem) < 8 for sem, _, _, _ in batch_items):
+            return [
+                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
+                for sem, glob, use_sw, trim in batch_items
+            ]
+
+        try:
+            batch_size = len(batch_items)
+            max_sem_len = max(len(sem) for sem, _, _, _ in batch_items)
+
+            semantic_batch = torch.zeros(
+                (batch_size, max_sem_len),
+                dtype=torch.long,
+                device=self.device,
+            )
+            global_batch = torch.zeros(
+                (batch_size, 32),
+                dtype=torch.long,
+                device=self.device,
+            )
+            semantic_lengths: List[int] = []
+
+            for idx, (semantic_ids, global_ids, _, _) in enumerate(batch_items):
+                sem_len = len(semantic_ids)
+                semantic_lengths.append(sem_len)
+                semantic_batch[idx, :sem_len] = torch.tensor(
+                    semantic_ids,
+                    dtype=torch.long,
+                    device=self.device,
+                )
+                global_batch[idx] = torch.tensor(
+                    self._normalize_global_ids(global_ids),
+                    dtype=torch.long,
+                    device=self.device,
+                )
+
+            with torch.inference_mode():
+                wav_batch = self.audio_tokenizer.detokenize(global_batch, semantic_batch)
+
+            results: List[Optional[bytes]] = []
+            for idx, sem_len in enumerate(semantic_lengths):
+                audio = wav_batch[idx] if getattr(wav_batch, "ndim", 1) > 1 else wav_batch
+                if isinstance(audio, torch.Tensor):
+                    audio = audio.detach().cpu().numpy()
+                audio_np = np.asarray(audio).reshape(-1)
+                expected_samples = sem_len * 320
+                if expected_samples > 0 and len(audio_np) > expected_samples:
+                    audio_np = audio_np[:expected_samples]
+                audio_np = np.clip(audio_np, -1.0, 1.0)
+                audio_int16 = (audio_np * 32767).astype(np.int16)
+                results.append(audio_int16.tobytes())
+            return results
+        except Exception:
+            logger.exception("BiCodec batched decode failed; falling back to sequential decode")
+            return [
+                self.decode_streaming(sem, glob, use_sliding_window=use_sw, trim_warmup=trim)
+                for sem, glob, use_sw, trim in batch_items
+            ]
+
+    def _ensure_batch_worker(self, loop: asyncio.AbstractEventLoop) -> asyncio.Queue:
+        key = id(loop)
+        with self._batch_map_lock:


**Extracting line numbers for changes**


nl -ba veena3modal/core/bicodec_decoder.py | sed -n '100,220p'


nl -ba veena3modal/core/bicodec_decoder.py | sed -n '340,470p'


nl -ba veena3modal/core/bicodec_decoder.py | sed -n '470,560p'