Output:
Chunk ID: 1d1df6
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1429
Output:
            timing={},
            gpu_samples={
                "start": gpu_start,
                "end": _query_gpu_snapshot(),
            },
        )
    except Exception as e:
        return RequestResult(
            success=False,
            status_code=0,
            latency_ms=(time.time() - start) * 1000,
            error=str(e)[:200],
            speaker=speaker,
            text_length=len(text),
            stream=stream,
            timing={},
            gpu_samples={
                "start": gpu_start,
                "end": _query_gpu_snapshot(),
            },
        )

async def run_level(
    base_url: str,
    num_requests: int,
    concurrency: int,
    text_category: str,
    chunking: bool,
    stream: bool,
    max_tokens: int,
    request_timeout: float,
    gpu_monitor: Optional[GPUMonitor] = None,
) -> LevelResult:
    mode = "streaming" if stream else "non-streaming"
    level = LevelResult(
        level_name=f"{concurrency}c-{text_category}-{mode}",
        concurrency=concurrency,
        total_requests=num_requests,
        mode=mode,
        text_category=text_category,
        chunking=chunking,
    )

    semaphore = asyncio.Semaphore(concurrency)

    if text_category == "mixed":
        texts = list(TEST_TEXTS.values())
    else:
        texts = [TEST_TEXTS.get(text_category, TEST_TEXTS["short"])]

    async def limited_request(i: int, client: httpx.AsyncClient) -> RequestResult:
        async with semaphore:
            text = texts[i % len(texts)]
            speaker = SPEAKERS[i % len(SPEAKERS)]
            return await make_request(
                client=client,
                base_url=base_url,
                text=text,
                speaker=speaker,
                chunking=chunking,
                max_tokens=max_tokens,
                stream=stream,
                timeout=request_timeout,
            )

    if gpu_monitor:
        gpu_monitor.snapshots.clear()

    start_time = time.time()
    limits = httpx.Limits(
        max_connections=concurrency + 10,
        max_keepalive_connections=concurrency,
    )
    async with httpx.AsyncClient(limits=limits) as client:
        tasks = [limited_request(i, client) for i in range(num_requests)]
        responses = await asyncio.gather(*tasks, return_exceptions=True)

    level.wall_time_seconds = time.time() - start_time

    for item in responses:
        if isinstance(item, RequestResult):
            level.results.append(item)
        else:
            level.results.append(
                RequestResult(
                    success=False,
                    status_code=0,
                    latency_ms=0,
                    error=str(item)[:200],
                    stream=stream,
                )
            )

    if gpu_monitor:
        level.gpu_summary = gpu_monitor.summary()

    return level

def print_level_report(level: LevelResult):
    print(f"\n{'=' * 72}")
    print(f"  {level.level_name.upper()}")
    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}, chunking={level.chunking}")
    print(f"{'=' * 72}")

    print(f"  Success:     {len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"  Wall time:   {level.wall_time_seconds:.2f}s")
    print(f"  Throughput:  {level.throughput_rps:.2f} req/s")
    print(f"  Audio total: {level.audio_duration_total:.2f}s")
    print(f"  Eff. RTF:    {level.effective_rtf:.3f}")

    if level.observed_inflight_max() is not None:
        print(f"  Peak inflight: {level.observed_inflight_max()}")

    latency = level.latency_stats
    if latency:
        print("\n  Latency (ms):")
        print(
            f"    min={latency['min']:.0f} avg={latency['avg']:.0f} "
            f"p50={latency['p50']:.0f} p95={latency['p95']:.0f} p99={latency['p99']:.0f} max={latency['max']:.0f}"
        )

    ttfb = level.ttfb_stats
    if ttfb and not level.mode.startswith("stream"):
        print("\n  Server TTFB (ms):")
        print(
            f"    min={ttfb['min']:.0f} avg={ttfb['avg']:.0f} "
            f"p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}"
        )

    if level.mode == "non-streaming":
        llm = level.timing_stats("generation_ms")
        if llm:
            print("\n  Generation (ms):")
            print(f"    min={llm['min']:.0f} avg={llm['avg']:.0f} p50={llm['p50']:.0f} p95={llm['p95']:.0f} max={llm['max']:.0f}")
        wall = level.timing_stats("llm_batch_wall_ms_total")
        if wall:
            print("\n  LLM batch wall (ms):")
            print(f"    min={wall['min']:.0f} avg={wall['avg']:.0f} p50={wall['p50']:.0f} p95={wall['p95']:.0f} max={wall['max']:.0f}")
        gpu = level.timing_stats("llm_batch_gpu_ms_total")
        if gpu:
            print("\n  LLM batch GPU (ms):")
            print(f"    min={gpu['min']:.0f} avg={gpu['avg']:.0f} p50={gpu['p50']:.0f} p95={gpu['p95']:.0f} max={gpu['max']:.0f}")
        parse = level.timing_stats("llm_parse_ms")
        if parse:
            print("\n  Parse (ms):")
            print(f"    avg={parse['avg']:.0f} p95={parse['p95']:.0f}")
        decode = level.timing_stats("llm_decode_wall_ms_total")
        if decode:
            print("\n  Decode (ms):")
            print(f"    avg={decode['avg']:.0f} p95={decode['p95']:.0f}")

        token_ms, token_count = level.timing_rate("generation_ms", "llm_token_total")
        if token_count:
            print(f"  Token timing: avg {token_ms / token_count * 1000:.3f} us/token")

    if level.gpu_summary and "memory_mb" in level.gpu_summary:
        gpu_summary = level.gpu_summary
        mem = gpu_summary["memory_mb"]
        util = gpu_summary["gpu_util_pct"]


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '760,920p' scripts/stress_test_optimized_local.py


Output:
Chunk ID: 0a5af3
Wall time: 0.0501 seconds
Process exited with code 0
Original token count: 1554
Output:
        util = gpu_summary["gpu_util_pct"]
        temp = gpu_summary["temperature_c"]
        print("\n  GPU:")
        print(f"    Memory: {mem['avg']:.0f}MB avg, {mem['max']:.0f}MB peak / {mem['total']:.0f}MB")
        print(f"    Util:   {util['avg']:.0f}% avg, {util['max']:.0f}% peak")
        print(f"    Temp:   {temp['avg']:.0f}C avg, {temp['max']:.0f}C peak")

    errors = level.error_summary()
    if errors:
        print("\n  Errors:")
        for key, count in sorted(errors.items(), key=lambda kv: -kv[1]):
            print(f"    [{count}x] {key}")

def print_summary_table(levels: List[LevelResult]):
    print(f"\n{'=' * 110}")
    print("  STRESS BENCHMARK SUMMARY")
    print(f"{'=' * 110}")
    header = f"{'Level':<26} {'OK%':>5} {'RPS':>7} {'p50':>7} {'p95':>7} {'GPU%':>5} {'GPUmem':>8} {'RTF':>6} {'Inflight':>8}"
    print(header)
    print("-" * 110)

    for level in levels:
        lat = level.latency_stats
        p50 = f"{lat.get('p50', 0):.0f}" if lat else "---"
        p95 = f"{lat.get('p95', 0):.0f}" if lat else "---"
        gpu_util = level.gpu_summary.get("gpu_util_pct", {}).get("avg", 0)
        gpu_mem = level.gpu_summary.get("memory_mb", {}).get("max", 0)
        inflight = level.observed_inflight_max() or 0
        print(
            f"{level.level_name:<26} "
            f"{level.success_rate:>4.0%} "
            f"{level.throughput_rps:>7.2f} "
            f"{p50:>7} "
            f"{p95:>7} "
            f"{gpu_util:>4.0f}% "
            f"{gpu_mem:>7.0f}MB "
            f"{level.effective_rtf:>6.3f} "
            f"{inflight:>8d}"
        )

    print("=" * 110)

def save_results(levels: List[LevelResult], path: str):
    payload = []
    for level in levels:
        payload.append({
            "level": level.level_name,
            "concurrency": level.concurrency,
            "mode": level.mode,
            "chunking": level.chunking,
            "text_category": level.text_category,
            "success_rate": level.success_rate,
            "throughput_rps": level.throughput_rps,
            "wall_time_s": level.wall_time_seconds,
            "latency_ms": level.latency_stats,
            "ttfb_ms": level.ttfb_stats,
            "effective_rtf": level.effective_rtf,
            "audio_total_seconds": level.audio_duration_total,
            "observed_inflight_max": level.observed_inflight_max(),
            "errors": level.error_summary(),
            "gpu": level.gpu_summary,
            "timing_averages": {
                "generation_ms": level.timing_stats("generation_ms"),
                "llm_batch_wall_ms_total": level.timing_stats("llm_batch_wall_ms_total"),
                "llm_batch_gpu_ms_total": level.timing_stats("llm_batch_gpu_ms_total"),
                "llm_parse_ms": level.timing_stats("llm_parse_ms"),
                "llm_decode_wall_ms_total": level.timing_stats("llm_decode_wall_ms_total"),
            },
            "requests": [
                {
                    "request_id": r.request_id,
                    "success": r.success,
                    "status_code": r.status_code,
                    "latency_ms": r.latency_ms,
                    "ttfb_ms": r.ttfb_ms,
                    "server_ttfb_ms": r.server_ttfb_ms,
                    "audio_bytes": r.audio_bytes,
                    "audio_seconds": r.audio_seconds,
                    "request_inflight": r.request_inflight,
                    "error": r.error,
                    "timing": r.timing,
                }
                for r in level.results
            ],
        })

    with open(path, "w") as fd:
        json.dump(payload, fd, indent=2)

async def run_stress_test(args):
    base_url = args.url

    try:
        async with httpx.AsyncClient() as client:
            response = await client.get(f"{base_url}/v1/tts/health", timeout=10)
            health = response.json()
            print(f"  Health: {health.get('status')}")
            print(f"  Version: {health.get('model_version')}")
            print(f"  GPU: {health.get('gpu_available')}")
    except Exception as e:
        print(f"ERROR: server not reachable at {base_url}: {e}")
        sys.exit(1)

    levels = [int(v) for v in args.levels.split(",") if v.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    mode_label = "streaming" if args.stream else "non-streaming"
    results: List[LevelResult] = []

    print(f"\nWarming up (1 request, non-streaming)...")
    warmup = await run_level(
        base_url=base_url,
        num_requests=1,
        concurrency=1,
        text_category=args.text,
        chunking=args.chunking,
        stream=False,
        max_tokens=args.max_tokens,
        request_timeout=args.timeout_seconds,
        gpu_monitor=gpu_monitor,
    )
    print(f"  warmup ok: {warmup.success_rate:.0%}")

    for conc in levels:
        num_requests = max(conc * 3, 10)
        num_requests = min(num_requests, 200)
        level = await run_level(
            base_url=base_url,
            num_requests=num_requests,
            concurrency=conc,
            text_category=args.text,
            chunking=args.chunking,
            stream=args.stream,
            max_tokens=args.max_tokens,
            request_timeout=args.timeout_seconds,
            gpu_monitor=gpu_monitor,
        )
        print_level_report(level)
        results.append(level)
        if conc != levels[-1]:
            print("  cooldown 3s...")
            await asyncio.sleep(3)

    gpu_monitor.stop()
    print_summary_table(results)

    if args.output:
        save_results(results, args.output)
        print(f"\nResults written to {args.output}")

def main():
    parser = argparse.ArgumentParser(description="Detailed local optimized-path TTS benchmark")
    parser.add_argument("--url", default=DEFAULT_URL, help=f"Server URL (default: {DEFAULT_URL})")
    parser.add_argument("--levels", default="1,5,10,20,50", help="Comma-separated concurrency levels")
    parser.add_argument("--text", default="short", choices=["short", "medium", "long", "mixed"], help="Input text category")
    parser.add_argument("--chunking", action="store_true", help="Enable server-side chunking", default=True)


Summary: auto


sed -n '340,450p' scripts/stress_test_optimized_local.py


Output:
Chunk ID: b0126f
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 1094
Output:

def _parse_perf_headers(headers: Any) -> Dict[str, Any]:
    """Parse server timing headers into numeric metric fields."""
    metrics: Dict[str, Any] = {}

    # Explicit compact payload.
    detail_payload = headers.get("x-perf-details") if hasattr(headers, "get") else None
    if detail_payload:
        try:
            parsed = json.loads(detail_payload)
            if isinstance(parsed, dict):
                metrics.update(parsed)
        except Exception:
            pass

    header_map = {
        "x-llm-token-total": "llm_token_total",
        "x-llm-batch-count": "llm_batch_count",
        "x-llm-batch-wall-ms": "llm_batch_wall_ms_total",
        "x-llm-batch-gpu-ms": "llm_batch_gpu_ms_total",
        "x-llm-batch-wall-ms-min": "llm_batch_wall_ms_min",
        "x-llm-batch-wall-ms-max": "llm_batch_wall_ms_max",
        "x-llm-batch-wall-ms-p50": "llm_batch_wall_ms_p50",
        "x-llm-batch-gpu-ms-min": "llm_batch_gpu_ms_min",
        "x-llm-batch-gpu-ms-max": "llm_batch_gpu_ms_max",
        "x-llm-batch-gpu-ms-p50": "llm_batch_gpu_ms_p50",
        "x-llm-decode-wall-ms": "llm_decode_wall_ms_total",
        "x-llm-decode-gpu-ms": "llm_decode_gpu_ms_total",
        "x-llm-decode-wall-ms-min": "llm_decode_wall_ms_min",
        "x-llm-decode-wall-ms-max": "llm_decode_wall_ms_max",
        "x-llm-decode-wall-ms-p50": "llm_decode_wall_ms_p50",
        "x-llm-decode-gpu-ms-min": "llm_decode_gpu_ms_min",
        "x-llm-decode-gpu-ms-max": "llm_decode_gpu_ms_max",
        "x-llm-decode-gpu-ms-p50": "llm_decode_gpu_ms_p50",
        "x-llm-decode-calls": "decode_calls",
        "x-llm-decode-cpu-ms": "llm_decode_cpu_ms",
        "x-llm-parse-ms": "llm_parse_ms",
        "x-llm-parse-avg-ms": "llm_parse_ms_avg",
        "x-llm-tokens-per-batch-min": "tokens_per_batch_min",
        "x-llm-tokens-per-batch-max": "tokens_per_batch_max",
        "x-llm-tokens-per-batch-p50": "tokens_per_batch_p50",
        "x-generation-ms": "generation_ms",
        "x-request-inflight": "request_inflight",
        "x-llm-text-chunked": "text_chunked",
        "x-llm-chunks-processed": "chunks_processed",
        "x-llm-sr-applied": "sr_applied",
        "x-llm-time-per-token-ms": "llm_time_per_token_ms",
        "x-llm-time-per-batch-wall-ms": "llm_time_per_batch_wall_ms",
        "x-llm-time-per-batch-gpu-ms": "llm_time_per_batch_gpu_ms",
    }

    bool_keys = {"text_chunked", "sr_applied"}
    int_keys = {
        "llm_token_total",
        "llm_batch_count",
        "decode_calls",
        "chunks_processed",
        "request_inflight",
    }

    if hasattr(headers, "items"):
        for raw_key, raw_val in headers.items():
            lk = str(raw_key).lower()
            target_key = header_map.get(lk)
            if target_key is None:
                continue
            if target_key in bool_keys:
                b = _coerce_bool(raw_val)
                if b is not None:
                    metrics[target_key] = b
                continue

            v = _coerce_number(raw_val)
            if v is None:
                continue
            if target_key in int_keys and float(v).is_integer():
                metrics[target_key] = int(v)
            else:
                metrics[target_key] = int(v) if float(v).is_integer() else float(v)

    if "llm_token_total" in metrics and "generation_ms" in metrics and metrics["llm_token_total"]:
        try:
            metrics["llm_time_per_token_ms"] = float(metrics["generation_ms"]) / float(metrics["llm_token_total"])
        except Exception:
            pass

    if "llm_batch_count" in metrics and "generation_ms" in metrics and metrics["llm_batch_count"]:
        try:
            metrics["llm_time_per_batch_wall_ms"] = float(metrics["generation_ms"]) / float(metrics["llm_batch_count"])
        except Exception:
            pass

    if "llm_batch_count" in metrics and "llm_batch_gpu_ms_total" in metrics and metrics["llm_batch_count"]:
        try:
            metrics["llm_time_per_batch_gpu_ms"] = float(metrics["llm_batch_gpu_ms_total"]) / float(metrics["llm_batch_count"])
        except Exception:
            pass

    return metrics

async def make_request(
    client: httpx.AsyncClient,
    base_url: str,
    text: str,
    speaker: str,
    chunking: bool,
    max_tokens: int,
    stream: bool = False,
    timeout: float = 120.0,
) -> RequestResult:


Input:
*** Begin Patch
*** Update File: scripts/stress_test_optimized_local.py
@@
 class RequestResult:
@@
     stream: bool = False
     timing: Dict[str, Any] = field(default_factory=dict)
     request_inflight: Optional[int] = None
+    client_inflight: Optional[int] = None
     gpu_samples: Dict[str, Any] = field(default_factory=dict)
@@
 class LevelResult:
@@
     mode: str
     text_category: str
     chunking: bool
+    client_inflight_peak: Optional[int] = None
     results: List[RequestResult] = field(default_factory=list)
@@
     def observed_inflight_max(self) -> Optional[int]:
-        values = [r.request_inflight for r in self.successes if isinstance(r.request_inflight, int)]
+        values = [r.request_inflight for r in self.successes if isinstance(r.request_inflight, int)]
+        values.extend([r.client_inflight for r in self.successes if isinstance(r.client_inflight, int)])
         if not values:
             return None
         return max(values)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_optimized_local.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: scripts/stress_test_optimized_local.py
@@
         "x-llm-tokens-per-batch-min": "tokens_per_batch_min",
         "x-llm-tokens-per-batch-max": "tokens_per_batch_max",
         "x-llm-tokens-per-batch-p50": "tokens_per_batch_p50",
         "x-generation-ms": "generation_ms",
+        "x-llm-generation-ms": "generation_ms",
         "x-request-inflight": "request_inflight",
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_optimized_local.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


Input:
*** Begin Patch
*** Update File: scripts/stress_test_optimized_local.py
@@
 async def run_level(
     base_url: str,
     num_requests: int,
     concurrency: int,
@@
     level = LevelResult(
         level_name=f"{concurrency}c-{text_category}-{mode}",
         concurrency=concurrency,
         total_requests=num_requests,
         mode=mode,
         text_category=text_category,
         chunking=chunking,
+        client_inflight_peak=0,
     )

     semaphore = asyncio.Semaphore(concurrency)
+    inflight_lock = asyncio.Lock()
+    active_inflight = 0
+    peak_inflight = 0

     if text_category == "mixed":
         texts = list(TEST_TEXTS.values())
     else:
         texts = [TEST_TEXTS.get(text_category, TEST_TEXTS["short"])]

     async def limited_request(i: int, client: httpx.AsyncClient) -> RequestResult:
+        nonlocal active_inflight, peak_inflight
         async with semaphore:
             text = texts[i % len(texts)]
             speaker = SPEAKERS[i % len(SPEAKERS)]
-            return await make_request(
-                client=client,
-                base_url=base_url,
-                text=text,
-                speaker=speaker,
-                chunking=chunking,
-                max_tokens=max_tokens,
-                stream=stream,
-                timeout=request_timeout,
-            )
+            async with inflight_lock:
+                active_inflight += 1
+                if active_inflight > peak_inflight:
+                    peak_inflight = active_inflight
+                local_inflight = active_inflight
+
+            try:
+                result = await make_request(
+                    client=client,
+                    base_url=base_url,
+                    text=text,
+                    speaker=speaker,
+                    chunking=chunking,
+                    max_tokens=max_tokens,
+                    stream=stream,
+                    timeout=request_timeout,
+                )
+                result.client_inflight = local_inflight
+                result.timing["client_inflight_start"] = local_inflight
+                return result
+            finally:
+                async with inflight_lock:
+                    active_inflight = max(0, active_inflight - 1)
@@
     for item in responses:
         if isinstance(item, RequestResult):
             level.results.append(item)
         else:
@@
                     stream=stream,
                 )
+    level.client_inflight_peak = peak_inflight
+    if level.client_inflight_peak:
+        level.client_inflight_peak = max(level.client_inflight_peak, 1)

     if gpu_monitor:
         level.gpu_summary = gpu_monitor.summary()
*** End Patch


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/veenaModal/scripts/stress_test_optimized_local.py:
                    stream=stream,
                )

    if gpu_monitor:
        level.gpu_summary = gpu_monitor.summary()


sed -n '620,710p' scripts/stress_test_optimized_local.py


Output:
Chunk ID: cd39b6
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 690
Output:
            gpu_samples={
                "start": gpu_start,
                "end": _query_gpu_snapshot(),
            },
        )

async def run_level(
    base_url: str,
    num_requests: int,
    concurrency: int,
    text_category: str,
    chunking: bool,
    stream: bool,
    max_tokens: int,
    request_timeout: float,
    gpu_monitor: Optional[GPUMonitor] = None,
) -> LevelResult:
    mode = "streaming" if stream else "non-streaming"
    level = LevelResult(
        level_name=f"{concurrency}c-{text_category}-{mode}",
        concurrency=concurrency,
        total_requests=num_requests,
        mode=mode,
        text_category=text_category,
        chunking=chunking,
    )

    semaphore = asyncio.Semaphore(concurrency)

    if text_category == "mixed":
        texts = list(TEST_TEXTS.values())
    else:
        texts = [TEST_TEXTS.get(text_category, TEST_TEXTS["short"])]

    async def limited_request(i: int, client: httpx.AsyncClient) -> RequestResult:
        async with semaphore:
            text = texts[i % len(texts)]
            speaker = SPEAKERS[i % len(SPEAKERS)]
            return await make_request(
                client=client,
                base_url=base_url,
                text=text,
                speaker=speaker,
                chunking=chunking,
                max_tokens=max_tokens,
                stream=stream,
                timeout=request_timeout,
            )

    if gpu_monitor:
        gpu_monitor.snapshots.clear()

    start_time = time.time()
    limits = httpx.Limits(
        max_connections=concurrency + 10,
        max_keepalive_connections=concurrency,
    )
    async with httpx.AsyncClient(limits=limits) as client:
        tasks = [limited_request(i, client) for i in range(num_requests)]
        responses = await asyncio.gather(*tasks, return_exceptions=True)

    level.wall_time_seconds = time.time() - start_time

    for item in responses:
        if isinstance(item, RequestResult):
            level.results.append(item)
        else:
            level.results.append(
                RequestResult(
                    success=False,
                    status_code=0,
                    latency_ms=0,
                    error=str(item)[:200],
                    stream=stream,
                )
            )

    if gpu_monitor:
        level.gpu_summary = gpu_monitor.summary()

    return level

def print_level_report(level: LevelResult):
    print(f"\n{'=' * 72}")
    print(f"  {level.level_name.upper()}")
    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}, chunking={level.chunking}")
    print(f"{'=' * 72}")

    print(f"  Success:     {len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")


Input:
*** Begin Patch
*** Update File: scripts/stress_test_optimized_local.py
@@
     level = LevelResult(
         level_name=f"{concurrency}c-{text_category}-{mode}",
         concurrency=concurrency,
         total_requests=num_requests,
         mode=mode,
         text_category=text_category,
         chunking=chunking,
     )

     semaphore = asyncio.Semaphore(concurrency)
+    inflight_lock = asyncio.Lock()
+    active_inflight = 0
+    peak_inflight = 0
@@
     async def limited_request(i: int, client: httpx.AsyncClient) -> RequestResult:
+        nonlocal active_inflight, peak_inflight
         async with semaphore:
             text = texts[i % len(texts)]
             speaker = SPEAKERS[i % len(SPEAKERS)]
-            return await make_request(
-                client=client,
-                base_url=base_url,
-                text=text,
-                speaker=speaker,
-                chunking=chunking,
-                max_tokens=max_tokens,
-                stream=stream,
-                timeout=request_timeout,
-            )
+            async with inflight_lock:
+                active_inflight += 1
+                if active_inflight > peak_inflight:
+                    peak_inflight = active_inflight
+                local_inflight = active_inflight
+
+            try:
+                result = await make_request(
+                    client=client,
+                    base_url=base_url,
+                    text=text,
+                    speaker=speaker,
+                    chunking=chunking,
+                    max_tokens=max_tokens,
+                    stream=stream,
+                    timeout=request_timeout,
+                )
+                result.client_inflight = local_inflight
+                result.timing["client_inflight_start"] = local_inflight
+                return result
+            finally:
+                async with inflight_lock:
+                    active_inflight = max(0, active_inflight - 1)
@@
     for item in responses:
         if isinstance(item, RequestResult):
             level.results.append(item)
         else:
             level.results.append(
                 RequestResult(
@@
                 )
             )
+    level.client_inflight_peak = peak_inflight
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_optimized_local.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


The script is now instrumented for client-side in-flight tracking. Next I’ll finish JSON capture of that concurrency signal and per-request GPU samples so the benchmark file has the full breakdown.


Input:
*** Begin Patch
*** Update File: scripts/stress_test_optimized_local.py
@@
             "errors": level.error_summary(),
             "gpu": level.gpu_summary,
+            "client_inflight_peak": level.client_inflight_peak,
             "timing_averages": {
                 "generation_ms": level.timing_stats("generation_ms"),
                 "llm_batch_wall_ms_total": level.timing_stats("llm_batch_wall_ms_total"),
                 "llm_batch_gpu_ms_total": level.timing_stats("llm_batch_gpu_ms_total"),
                 "llm_parse_ms": level.timing_stats("llm_parse_ms"),
@@
                     "ttfb_ms": r.ttfb_ms,
                     "server_ttfb_ms": r.server_ttfb_ms,
                     "audio_bytes": r.audio_bytes,
                     "audio_seconds": r.audio_seconds,
                     "request_inflight": r.request_inflight,
+                    "client_inflight": r.client_inflight,
                     "error": r.error,
                     "timing": r.timing,
+                    "gpu_samples": r.gpu_samples,
                 }
                 for r in level.results
             ],
         })
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_optimized_local.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


I’ve finished the instrumentation changes. I’m now running a focused local stress run (1, 2, and 4 concurrent, short text, low token cap) and an output JSON so you get per-request timing + GPU deltas in one pass.


python3 scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 64 --timeout-seconds 8 --no-chunking --output /tmp/bench_short_optimized_path.json


Output:
Chunk ID: 1c875c
Wall time: 9.1054 seconds
Process exited with code 0
Original token count: 691
Output:
  Health: healthy
  Version: LLM
  GPU: True

Warming up (1 request, non-streaming)...
  warmup ok: 0%

========================================================================
  1C-SHORT-NON-STREAMING
  10 requests, 1 concurrent, non-streaming, chunking=False
========================================================================
  Success:     0/10 (0%)
  Wall time:   0.77s
  Throughput:  13.01 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   0% avg, 0% peak
    Temp:   30C avg, 30C peak

  Errors:
    [10x] {"error":{"code":"VALIDATION_ERROR","message":"1 validation error for TTSGenerateRequest\nmax_tokens
  cooldown 3s...

========================================================================
  2C-SHORT-NON-STREAMING
  10 requests, 2 concurrent, non-streaming, chunking=False
========================================================================
  Success:     0/10 (0%)
  Wall time:   0.79s
  Throughput:  12.65 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   0% avg, 0% peak
    Temp:   30C avg, 30C peak

  Errors:
    [10x] {"error":{"code":"VALIDATION_ERROR","message":"1 validation error for TTSGenerateRequest\nmax_tokens
  cooldown 3s...

========================================================================
  4C-SHORT-NON-STREAMING
  12 requests, 4 concurrent, non-streaming, chunking=False
========================================================================
  Success:     0/12 (0%)
  Wall time:   0.93s
  Throughput:  12.94 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   0% avg, 0% peak
    Temp:   30C avg, 30C peak

  Errors:
    [12x] {"error":{"code":"VALIDATION_ERROR","message":"1 validation error for TTSGenerateRequest\nmax_tokens

==============================================================================================================
  STRESS BENCHMARK SUMMARY
==============================================================================================================
Level                        OK%     RPS     p50     p95  GPU%   GPUmem    RTF Inflight
--------------------------------------------------------------------------------------------------------------
1c-short-non-streaming       0%   13.01     ---     ---    0%   41871MB  0.000        0
2c-short-non-streaming       0%   12.65     ---     ---    0%   41871MB  0.000        0
4c-short-non-streaming       0%   12.94     ---     ---    0%   41871MB  0.000        0
==============================================================================================================

Results written to /tmp/bench_short_optimized_path.json


rg -n "class TTSGenerateRequest|max_tokens|generate request|max_new" veena3modal -g '*.py'


Output:
Chunk ID: b97a10
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 1608
Output:
veena3modal/services/tts_runtime.py:330:        max_tokens=128,  # Only need ~60 tokens (32 global + some semantic)
veena3modal/services/tts_runtime.py:397:    max_tokens: int = 4096,
veena3modal/services/tts_runtime.py:412:        max_tokens: Maximum tokens to generate
veena3modal/services/tts_runtime.py:460:                "max_tokens": max_tokens,
veena3modal/services/tts_runtime.py:472:            max_tokens=max_tokens,
veena3modal/services/tts_runtime.py:666:    max_tokens: int = 4096,
veena3modal/services/tts_runtime.py:737:                    max_tokens=max_tokens,
veena3modal/services/tts_runtime.py:756:                    max_tokens=max_tokens,
veena3modal/services/tts_runtime.py:771:                max_tokens=max_tokens,
veena3modal/services/tts_runtime.py:884:    max_tokens: int = 4096,
veena3modal/services/tts_runtime.py:903:        max_tokens: Maximum tokens to generate
veena3modal/services/tts_runtime.py:984:                max_tokens,
veena3modal/services/tts_runtime.py:1021:                    max_tokens=max_tokens,
veena3modal/services/tts_runtime.py:1034:                    max_tokens=max_tokens,
veena3modal/services/tts_runtime.py:1079:    max_tokens: int,
veena3modal/services/tts_runtime.py:1118:                max_tokens=max_tokens,
veena3modal/services/tts_runtime.py:1147:                    max_tokens=max_tokens,
veena3modal/services/tts_runtime.py:1170:                    max_tokens=max_tokens,
veena3modal/services/sentence_store.py:57:    max_tokens: int = 4096,
veena3modal/services/sentence_store.py:76:        max_tokens: Maximum tokens to generate
veena3modal/services/sentence_store.py:96:        "max_tokens": max_tokens,
veena3modal/services/sentence_store.py:152:        max_tokens: int = 4096,
veena3modal/services/sentence_store.py:182:                max_tokens=max_tokens,
veena3modal/core/pipeline.py:86:        max_tokens: int = DEFAULT_MAX_TOKENS,
veena3modal/core/pipeline.py:104:            max_tokens: Max BiCodec tokens to generate (default: 2048)
veena3modal/core/pipeline.py:120:            max_tokens=max_tokens,
veena3modal/core/pipeline.py:154:                    "max_tokens": max_tokens,
veena3modal/core/pipeline.py:370:        max_tokens: int = DEFAULT_MAX_TOKENS,
veena3modal/core/pipeline.py:392:            max_tokens: Max BiCodec tokens to generate
veena3modal/core/pipeline.py:407:            max_tokens=max_tokens,
veena3modal/core/streaming_pipeline.py:102:        max_tokens: int = DEFAULT_MAX_TOKENS,
veena3modal/core/streaming_pipeline.py:116:            max_tokens: Max SNAC tokens to generate
veena3modal/core/streaming_pipeline.py:134:            max_tokens=max_tokens,
veena3modal/core/streaming_pipeline.py:231:        max_tokens: int = DEFAULT_MAX_TOKENS,
veena3modal/core/streaming_pipeline.py:251:            max_tokens: Max SNAC tokens to generate
veena3modal/core/streaming_pipeline.py:276:        # which doesn't exist in the Spark vocab and would cause generation to run to max_tokens
veena3modal/core/streaming_pipeline.py:281:            max_tokens=max_tokens,
veena3modal/core/streaming_pipeline.py:591:        max_tokens: int = DEFAULT_MAX_TOKENS,
veena3modal/core/streaming_pipeline.py:618:            max_tokens: Max tokens to generate
veena3modal/core/streaming_pipeline.py:654:            max_tokens=max_tokens,
veena3modal/core/streaming_pipeline.py:954:        max_tokens: int = DEFAULT_MAX_TOKENS,
veena3modal/core/streaming_pipeline.py:980:            max_tokens: Max tokens to generate
veena3modal/core/streaming_pipeline.py:1026:            max_tokens=max_tokens,
veena3modal/api/schemas.py:154:class TTSGenerateRequest(BaseModel):
veena3modal/api/schemas.py:231:    max_tokens: int = Field(
veena3modal/api/websocket_handler.py:75:    max_tokens: int = 4096
veena3modal/api/websocket_handler.py:92:            max_tokens=data.get("max_tokens", 4096),
veena3modal/api/websocket_handler.py:394:                max_tokens=req.max_tokens,
veena3modal/api/fastapi_app.py:452:                    max_tokens=req.max_tokens,
veena3modal/api/fastapi_app.py:466:                    max_tokens=req.max_tokens,
veena3modal/api/fastapi_app.py:527:                max_tokens=req.max_tokens,
veena3modal/api/fastapi_app.py:689:                    max_tokens=req.max_tokens,
veena3modal/api/fastapi_app.py:721:                            max_tokens=req.max_tokens,
veena3modal/processing/long_text_processor.py:149:        max_tokens: int = 4096,
veena3modal/processing/long_text_processor.py:169:            max_tokens: Max tokens per chunk
veena3modal/processing/long_text_processor.py:223:            # Calculate appropriate max_tokens for this chunk
veena3modal/processing/long_text_processor.py:227:            chunk_max_tokens = min(
veena3modal/processing/long_text_processor.py:231:            chunk_max_tokens = max(chunk_max_tokens, 100)  # Ensure minimum
veena3modal/processing/long_text_processor.py:240:                max_tokens=chunk_max_tokens,  # Use calculated limit, not global max_tokens
veena3modal/processing/long_text_processor.py:403:        max_tokens: int = 4096,
veena3modal/processing/long_text_processor.py:421:            max_tokens: Max tokens per chunk
veena3modal/processing/long_text_processor.py:450:                max_tokens=max_tokens,
veena3modal/tests/integration/test_sentence_store.py:65:            max_tokens=2048,
veena3modal/tests/unit/test_schemas.py:136:            max_tokens=2048,
veena3modal/tests/unit/test_schemas.py:355:    def test_max_tokens_bounds(self):
veena3modal/tests/unit/test_schemas.py:356:        """max_tokens should be bounded 128-4096."""
veena3modal/tests/unit/test_schemas.py:357:        TTSGenerateRequest(text="Hi", speaker="lipakshi", max_tokens=128)
veena3modal/tests/unit/test_schemas.py:358:        TTSGenerateRequest(text="Hi", speaker="lipakshi", max_tokens=4096)
veena3modal/tests/unit/test_schemas.py:361:            TTSGenerateRequest(text="Hi", speaker="lipakshi", max_tokens=127)
veena3modal/tests/unit/test_schemas.py:363:            TTSGenerateRequest(text="Hi", speaker="lipakshi", max_tokens=4097)
veena3modal/tests/unit/test_websocket_handler.py:43:            "max_tokens": 2048,
veena3modal/tests/unit/test_websocket_handler.py:58:        assert req.max_tokens == 2048
veena3modal/tests/unit/test_sentence_store.py:87:            max_tokens=4096,
veena3modal/tests/unit/test_sentence_store.py:103:        assert record["max_tokens"] == 4096


The benchmark currently fails at request validation. I found `max_tokens` appears in the schema; I’m checking its field constraints now so we can run a valid load with real generation timing.


sed -n '130,190p' veena3modal/api/schemas.py


Output:
Chunk ID: 976c31
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 360
Output:
        return bracket_tag  # Keep as-is

    return re.sub(bracket_pattern, replace_bracket, text)

# === Enums ===

class AudioFormat(str, Enum):
    """Supported audio output formats."""
    WAV = "wav"
    OPUS = "opus"
    MP3 = "mp3"
    MULAW = "mulaw"
    FLAC = "flac"

class OutputSampleRate(str, Enum):
    """Output sample rate options."""
    SR_16KHZ = "16khz"
    SR_48KHZ = "48khz"

# === Request Schema ===

class TTSGenerateRequest(BaseModel):
    """
    Request schema for /v1/tts/generate.

    Mirrors validation rules from veena3srv/apps/api/serializers.py.
    """

    # Required
    text: str = Field(
        ...,
        min_length=1,
        max_length=MAX_TEXT_LENGTH,
        description="Text to synthesize (max 50,000 characters)"
    )

    # Speaker (required for indic_speakers model)
    speaker: Optional[str] = Field(
        None,
        max_length=50,
        description="Speaker name (internal or friendly)"
    )

    # Voice design (not currently supported)
    description: Optional[str] = Field(
        None,
        max_length=2000,
        description="Voice description (for voiceDesign model, NOT currently supported)"
    )
    voice_id: Optional[UUID] = Field(
        None,
        description="UUID of pre-created voice profile (for voiceDesign model)"
    )

    # Generation options
    model: Optional[str] = Field(
        None,
        max_length=50,


sed -n '190,280p' veena3modal/api/schemas.py


Output:
Chunk ID: dc66bf
Wall time: 0.0505 seconds
Process exited with code 0
Original token count: 631
Output:
        max_length=50,
        description="Model to use (optional). Default: indic_speakers"
    )
    seed: Optional[int] = Field(
        None,
        ge=0,
        le=2**31 - 1,
        description="Random seed for reproducibility (0-2147483647)"
    )
    stream: bool = Field(
        False,
        description="Stream audio as it's generated"
    )
    format: AudioFormat = Field(
        AudioFormat.WAV,
        description="Audio output format"
    )
    sample_rate: int = Field(
        16000,
        description="Sample rate in Hz (default: 16000)"
    )

    # Advanced parameters
    temperature: float = Field(
        0.4,
        ge=0.0,
        le=2.0,
        description="Sampling temperature (0.0-2.0)"
    )
    top_k: int = Field(
        50,
        ge=1,
        le=100,
        description="Top-k sampling (1-100)"
    )
    top_p: float = Field(
        1.0,
        ge=0.0,
        le=1.0,
        description="Nucleus sampling (0.0-1.0)"
    )
    max_tokens: int = Field(
        4096,
        ge=128,
        le=4096,
        description="Maximum BiCodec tokens to generate"
    )
    repetition_penalty: float = Field(
        1.05,
        ge=1.0,
        le=2.0,
        description="Repetition penalty (1.0-2.0)"
    )

    # Preprocessing toggles
    normalize: bool = Field(
        True,
        description="Apply text normalization before TTS"
    )
    normalize_verbose: bool = Field(
        False,
        description="Return normalized text in X-Normalized-Text header"
    )
    chunking: bool = Field(
        True,
        description="Enable intelligent text chunking for long inputs"
    )

    # Output options
    output: OutputSampleRate = Field(
        OutputSampleRate.SR_16KHZ,
        description="Output sample rate: '16khz' or '48khz' (super-resolution)"
    )

    # Internal fields (populated during validation)
    _original_text: Optional[str] = None
    _resolved_speaker: Optional[str] = None

    model_config = ConfigDict(use_enum_values=True)  # Serialize enums as strings

    @field_validator('text')
    @classmethod
    def validate_text(cls, v: str) -> str:
        """Validate text field: no empty, no control chars."""
        stripped = v.strip()
        if not stripped:
            raise ValueError("Text cannot be empty or whitespace only")

        # Check for control characters (except newlines/tabs)
        control_chars = [c for c in v if ord(c) < 32 and c not in '\n\t\r']
        if control_chars:


python3 scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 128 --timeout-seconds 8 --output /tmp/bench_short_optimized_valid.json


Output:
Chunk ID: af9e2c
Wall time: 30.0010 seconds
Process running with session ID 59335
Original token count: 0
Output:


I hit the max token validation issue; I’ve restarted the benchmark with `max_tokens=128` (schema minimum) and it’s running. I’ll collect final results from the running session now.


Output:
Chunk ID: 8baeb3
Wall time: 30.0007 seconds
Process running with session ID 59335
Original token count: 0
Output:


Output:
Chunk ID: 3c0d4c
Wall time: 30.0012 seconds
Process running with session ID 59335
Original token count: 0
Output:


Output:
Chunk ID: a2a3c0
Wall time: 30.0012 seconds
Process running with session ID 59335
Original token count: 0
Output:


Output: write_stdin failed: stdin is closed for this session; rerun exec_command with tty=true to keep stdin open


The previous run didn’t emit output mid-flight (likely buffered while waiting). I’m rerunning with unbuffered execution and a hard shell timeout so we get deterministic completion and captured logs.


timeout 180 python3 -u scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 128 --timeout-seconds 8 --output /tmp/bench_short_optimized_valid2.json


Output:
Chunk ID: c59a19
Wall time: 30.0007 seconds
Process running with session ID 81207
Original token count: 26
Output:
  Health: healthy
  Version: LLM
  GPU: True

Warming up (1 request, non-streaming)...
  warmup ok: 0%


Output:
Chunk ID: 42fd1f
Wall time: 30.0007 seconds
Process running with session ID 81207
Original token count: 0
Output:


Output:
Chunk ID: df0d70
Wall time: 30.0012 seconds
Process running with session ID 81207
Original token count: 127
Output:

========================================================================
  1C-SHORT-NON-STREAMING
  10 requests, 1 concurrent, non-streaming, chunking=True
========================================================================
  Success:     0/10 (0%)
  Wall time:   80.64s
  Throughput:  0.12 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   4% avg, 70% peak
    Temp:   28C avg, 31C peak

  Errors:
    [10x] TIMEOUT
  cooldown 3s...


Output:
Chunk ID: 0cb753
Wall time: 30.0013 seconds
Process running with session ID 81207
Original token count: 0
Output:


Output:
Chunk ID: 2455c4
Wall time: 30.0015 seconds
Process running with session ID 81207
Original token count: 127
Output:

========================================================================
  2C-SHORT-NON-STREAMING
  10 requests, 2 concurrent, non-streaming, chunking=True
========================================================================
  Success:     0/10 (0%)
  Wall time:   40.60s
  Throughput:  0.25 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   4% avg, 71% peak
    Temp:   28C avg, 30C peak

  Errors:
    [10x] TIMEOUT
  cooldown 3s...


Output:
Chunk ID: f54036
Wall time: 2.0963 seconds
Process exited with code 0
Original token count: 343
Output:

========================================================================
  4C-SHORT-NON-STREAMING
  12 requests, 4 concurrent, non-streaming, chunking=True
========================================================================
  Success:     0/12 (0%)
  Wall time:   24.68s
  Throughput:  0.49 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   4% avg, 76% peak
    Temp:   28C avg, 29C peak

  Errors:
    [12x] TIMEOUT

==============================================================================================================
  STRESS BENCHMARK SUMMARY
==============================================================================================================
Level                        OK%     RPS     p50     p95  GPU%   GPUmem    RTF Inflight
--------------------------------------------------------------------------------------------------------------
1c-short-non-streaming       0%    0.12     ---     ---    4%   41871MB  0.000        0
2c-short-non-streaming       0%    0.25     ---     ---    4%   41871MB  0.000        0
4c-short-non-streaming       0%    0.49     ---     ---    4%   41871MB  0.000        0
==============================================================================================================

Results written to /tmp/bench_short_optimized_valid2.json


The valid run completed. I’ll quickly inspect the result file to confirm per-request fields (timing + GPU samples + concurrency counters) are now persisted before I summarize bottlenecks.


python3 - <<'PY'
import json
p='/tmp/bench_short_optimized_valid2.json'
with open(p) as f:
    data=json.load(f)
print('levels', [x['level'] for x in data])
for x in data:
    req=x['requests'][0]
    print('level',x['level'],'has keys',sorted(list(req.keys())))
    print('sample keys', sorted((req.get('gpu_samples') or {}).keys()))
    print('timing keys sample', list(req.get('timing', {}).keys())[:8], '...')
    print('client_inflight_peak', x.get('client_inflight_peak'))
    print('request count', len(x['requests']), 'errors', x['errors'])
PY


Output:
Chunk ID: f9f2ad
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 298
Output:
levels ['1c-short-non-streaming', '2c-short-non-streaming', '4c-short-non-streaming']
level 1c-short-non-streaming has keys ['audio_bytes', 'audio_seconds', 'client_inflight', 'error', 'gpu_samples', 'latency_ms', 'request_id', 'request_inflight', 'server_ttfb_ms', 'status_code', 'success', 'timing', 'ttfb_ms']
sample keys ['end', 'start']
timing keys sample ['client_inflight_start'] ...
client_inflight_peak 1
request count 10 errors {'TIMEOUT': 10}
level 2c-short-non-streaming has keys ['audio_bytes', 'audio_seconds', 'client_inflight', 'error', 'gpu_samples', 'latency_ms', 'request_id', 'request_inflight', 'server_ttfb_ms', 'status_code', 'success', 'timing', 'ttfb_ms']
sample keys ['end', 'start']
timing keys sample ['client_inflight_start'] ...
client_inflight_peak 2
request count 10 errors {'TIMEOUT': 10}
level 4c-short-non-streaming has keys ['audio_bytes', 'audio_seconds', 'client_inflight', 'error', 'gpu_samples', 'latency_ms', 'request_id', 'request_inflight', 'server_ttfb_ms', 'status_code', 'success', 'timing', 'ttfb_ms']
sample keys ['end', 'start']
timing keys sample ['client_inflight_start'] ...
client_inflight_peak 4
request count 12 errors {'TIMEOUT': 12}


python3 -u scripts/stress_test_optimized_local.py --levels 1 --text short --max-tokens 128 --timeout-seconds 8 --no-chunking --output /tmp/bench_short_nochunk1.json


Output:
Chunk ID: b0085d
Wall time: 30.0008 seconds
Process running with session ID 76844
Original token count: 26
Output:
  Health: healthy
  Version: LLM
  GPU: True

Warming up (1 request, non-streaming)...
  warmup ok: 0%


Summary: auto


Output:
Chunk ID: 26a916
Wall time: 30.0013 seconds
Process running with session ID 76844
Original token count: 0
Output:


Output:
Chunk ID: 6df4b3
Wall time: 26.9697 seconds
Process exited with code 0
Original token count: 298
Output:

========================================================================
  1C-SHORT-NON-STREAMING
  10 requests, 1 concurrent, non-streaming, chunking=False
========================================================================
  Success:     0/10 (0%)
  Wall time:   80.63s
  Throughput:  0.12 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   4% avg, 72% peak
    Temp:   28C avg, 30C peak

  Errors:
    [10x] TIMEOUT

==============================================================================================================
  STRESS BENCHMARK SUMMARY
==============================================================================================================
Level                        OK%     RPS     p50     p95  GPU%   GPUmem    RTF Inflight
--------------------------------------------------------------------------------------------------------------
1c-short-non-streaming       0%    0.12     ---     ---    4%   41871MB  0.000        0
==============================================================================================================

Results written to /tmp/bench_short_nochunk1.json


python3 -u scripts/stress_test_optimized_local.py --levels 1 --text short --max-tokens 128 --timeout-seconds 8 --stream --output /tmp/bench_short_stream1.json


Output:
Chunk ID: 611ddb
Wall time: 30.0019 seconds
Process running with session ID 56049
Original token count: 26
Output:
  Health: healthy
  Version: LLM
  GPU: True

Warming up (1 request, non-streaming)...
  warmup ok: 0%


Output:
Chunk ID: 68be02
Wall time: 30.0017 seconds
Process running with session ID 56049
Original token count: 0
Output:


Output:
Chunk ID: 084c07
Wall time: 27.0973 seconds
Process exited with code 0
Original token count: 295
Output:

========================================================================
  1C-SHORT-STREAMING
  10 requests, 1 concurrent, streaming, chunking=True
========================================================================
  Success:     0/10 (0%)
  Wall time:   80.66s
  Throughput:  0.12 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   3% avg, 71% peak
    Temp:   28C avg, 29C peak

  Errors:
    [10x] TIMEOUT

==============================================================================================================
  STRESS BENCHMARK SUMMARY
==============================================================================================================
Level                        OK%     RPS     p50     p95  GPU%   GPUmem    RTF Inflight
--------------------------------------------------------------------------------------------------------------
1c-short-streaming           0%    0.12     ---     ---    3%   41871MB  0.000        0
==============================================================================================================

Results written to /tmp/bench_short_stream1.json


I now have comparable runs for non-streaming, streaming, and no-chunking paths; all are timing out at the current request-level limit. I’ll pull compact per-request fields from one result file so you can see what was captured before summarizing bottleneck signals.


python3 - <<'PY'
import json
from statistics import mean
for p in ['/tmp/bench_short_optimized_valid2.json','/tmp/bench_short_nochunk1.json','/tmp/bench_short_stream1.json']:
    with open(p) as f:
        data=json.load(f)
    print('\n',p)
    for level in data:
        reqs=level['requests']
        to=level['errors']
        lat=[r['latency_ms'] for r in reqs]
        print(level['level'],'success',sum(1 for r in reqs if r['success']),'/',len(reqs),'errors',to)
        print('  latency_ms avg/min/max', mean(lat) if lat else 0, min(lat), max(lat))
        if reqs:
            print('  first timing keys', sorted(reqs[0]['timing'].keys()) if reqs[0]['timing'] else [])
            print('  first gpu samples', reqs[0].get('gpu_samples'))
            print('  client inflight peak', level.get('client_inflight_peak'))
PY


Output:
Chunk ID: 83742f
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 607
Output:

 /tmp/bench_short_optimized_valid2.json
1c-short-non-streaming success 0 / 10 errors {'TIMEOUT': 10}
  latency_ms avg/min/max 8035.084939002991 8031.304836273193 8038.355112075806
  first timing keys ['client_inflight_start']
  first gpu samples {'start': {'gpu_utilization_pct': 69.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 29.0}, 'end': {'gpu_utilization_pct': 74.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 30.0}}
  client inflight peak 1
2c-short-non-streaming success 0 / 10 errors {'TIMEOUT': 10}
  latency_ms avg/min/max 8061.324191093445 8057.3389530181885 8066.410541534424
  first timing keys ['client_inflight_start']
  first gpu samples {'start': {'gpu_utilization_pct': 0.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 28.0}, 'end': {'gpu_utilization_pct': 0.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 28.0}}
  client inflight peak 2
4c-short-non-streaming success 0 / 12 errors {'TIMEOUT': 12}
  latency_ms avg/min/max 8113.631983598073 8092.6103591918945 8141.280174255371
  first timing keys ['client_inflight_start']
  first gpu samples {'start': {'gpu_utilization_pct': 0.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 28.0}, 'end': {'gpu_utilization_pct': 0.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 28.0}}
  client inflight peak 4

 /tmp/bench_short_nochunk1.json
1c-short-non-streaming success 0 / 10 errors {'TIMEOUT': 10}
  latency_ms avg/min/max 8033.7846755981445 8028.511047363281 8036.145448684692
  first timing keys ['client_inflight_start']
  first gpu samples {'start': {'gpu_utilization_pct': 0.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 28.0}, 'end': {'gpu_utilization_pct': 0.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 28.0}}
  client inflight peak 1

 /tmp/bench_short_stream1.json
1c-short-streaming success 0 / 10 errors {'TIMEOUT': 10}
  latency_ms avg/min/max 8037.485313415527 8032.725095748901 8041.086912155151
  first timing keys ['client_inflight_start']
  first gpu samples {'start': {'gpu_utilization_pct': 0.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 28.0}, 'end': {'gpu_utilization_pct': 0.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 28.0}}
  client inflight peak 1


python3 -u scripts/stress_test_optimized_local.py --levels 1 --text short --max-tokens 128 --timeout-seconds 35 --output /tmp/bench_short_timeout35_nonstream.json


Output:
Chunk ID: 8dbb2e
Wall time: 30.0003 seconds
Process running with session ID 41479
Original token count: 22
Output:
  Health: healthy
  Version: LLM
  GPU: True

Warming up (1 request, non-streaming)...


Output:
Chunk ID: fd7492
Wall time: 30.0016 seconds
Process running with session ID 41479
Original token count: 4
Output:
  warmup ok: 0%


Output:
Chunk ID: b20545
Wall time: 30.0004 seconds
Process running with session ID 41479
Original token count: 0
Output:


python3 - <<'PY'
import httpx, time
url='http://localhost:8000/v1/tts/generate'
payload={"text":"Hello, this is a quick test.","speaker":"lipakshi","stream":false,"chunking":true,"max_tokens":128,"format":"wav"}
start=time.time()
try:
    r=httpx.post(url,json=payload,timeout=35)
    print('status',r.status_code,'lat_ms', (time.time()-start)*1000)
    print('ttfb', r.headers.get('x-ttfb-ms'))
    print('gen', r.headers.get('x-generation-ms'))
    print('llm_batch_wall', r.headers.get('x-llm-batch-wall-ms'))
    print('llm_batch_gpu', r.headers.get('x-llm-batch-gpu-ms'))
    print('batch_count', r.headers.get('x-llm-batch-count'))
    print('tokens', r.headers.get('x-llm-token-total'))
    print('inflate', r.headers.get('x-request-inflight'))
    print('perf_len', len(r.content))
    print(r.text[:200])
except Exception as e:
    print('error',type(e),e)
PY


Output:
Chunk ID: 04e510
Wall time: 0.0640 seconds
Process exited with code 1
Original token count: 34
Output:
Traceback (most recent call last):
  File "<stdin>", line 3, in <module>
NameError: name 'false' is not defined. Did you mean: 'False'?


python3 - <<'PY'
import httpx, time
url='http://localhost:8000/v1/tts/generate'
payload={'text':'Hello, this is a quick test.','speaker':'lipakshi','stream':False,'chunking':True,'max_tokens':128,'format':'wav'}
start=time.time()
try:
    r=httpx.post(url,json=payload,timeout=35)
    print('status',r.status_code,'lat_ms', (time.time()-start)*1000)
    print('ttfb', r.headers.get('x-ttfb-ms'))
    print('gen', r.headers.get('x-generation-ms'))
    print('llm_batch_wall', r.headers.get('x-llm-batch-wall-ms'))
    print('llm_batch_gpu', r.headers.get('x-llm-batch-gpu-ms'))
    print('batch_count', r.headers.get('x-llm-batch-count'))
    print('tokens', r.headers.get('x-llm-token-total'))
    print('inflate', r.headers.get('x-request-inflight'))
    print('perf_details', r.headers.get('x-perf-details'))
    print('perf_exists', r.headers.get('x-perf-details') is not None)
    print('bytes', len(r.content))
    print('text', r.text[:220])
except Exception as e:
    print('error',type(e),e)
PY


Output:
Chunk ID: 9cbe5b
Wall time: 30.0015 seconds
Process running with session ID 90348
Original token count: 0
Output:


Output:
Chunk ID: 7eb7e3
Wall time: 4.0806 seconds
Process exited with code 0
Original token count: 11
Output:
error <class 'httpx.ReadTimeout'> timed out


python3 - <<'PY'
import httpx, time
url='http://localhost:8000/v1/tts/generate'
payload={'text':'Hello, this is a quick test.','speaker':'lipakshi','stream':False,'chunking':True,'max_tokens':128,'format':'wav'}
start=time.time()
timeout=httpx.Timeout(connect=20.0, read=40.0, write=20.0, pool=40.0)
try:
    r=httpx.post(url,json=payload,timeout=timeout)
    print('status',r.status_code,'lat_ms', (time.time()-start)*1000)
    print('ttfb', r.headers.get('x-ttfb-ms'))
    print('gen', r.headers.get('x-generation-ms'))
    print('llm_batch_wall', r.headers.get('x-llm-batch-wall-ms'))
    print('llm_batch_gpu', r.headers.get('x-llm-batch-gpu-ms'))
    print('batch_count', r.headers.get('x-llm-batch-count'))
    print('tokens', r.headers.get('x-llm-token-total'))
    print('inflate', r.headers.get('x-request-inflight'))
    print('perf_details', r.headers.get('x-perf-details'))
    print('bytes', len(r.content))
    print(r.text[:200])
except Exception as e:
    print('error',type(e),e)
PY


Output:
Chunk ID: 3ada1d
Wall time: 30.0012 seconds
Process running with session ID 35906
Original token count: 0
Output:


Output:
Chunk ID: 2d287d
Wall time: 9.1382 seconds
Process exited with code 0
Original token count: 11
Output:
error <class 'httpx.ReadTimeout'> timed out


timeout 60 python3 - <<'PY'
import httpx, time, json
url='http://localhost:8000/v1/tts/generate'
payload={'text':'Hello, this is a quick test.','speaker':'lipakshi','stream':False,'chunking':True,'max_tokens':128,'format':'wav'}
start=time.time()
try:
    r=httpx.post(url,json=payload,timeout=None)
    print('status',r.status_code,'lat_ms', (time.time()-start)*1000)
    print('ttfb', r.headers.get('x-ttfb-ms'))
    print('perf', r.headers.get('x-perf-details'))
    print('keys', [k for k in r.headers.keys() if k.lower().startswith('x-llm')][:10])
    print('bytes', len(r.content))
    print(r.text[:120])
except Exception as e:
    print('error',type(e),e)
PY


Output:
Chunk ID: 78d030
Wall time: 30.0015 seconds
Process running with session ID 92272
Original token count: 0
Output:


Output:
Chunk ID: abc41d
Wall time: 28.8934 seconds
Process exited with code 124
Original token count: 0
Output:


I’ve confirmed single-request hangs longer than 60s (no response, no completion headers), which aligns with the benchmark pattern. I’ll verify the final diff and then share a concise benchmark report + bottleneck readout.


git diff -- scripts/stress_test_optimized_local.py


Output:
Chunk ID: 281a12
Wall time: 0.0502 seconds
Process exited with code 0
Original token count: 0
Output:


git status --short scripts/stress_test_optimized_local.py


Output:
Chunk ID: 4614dd
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 11
Output:
?? scripts/stress_test_optimized_local.py


env | rg '^VEENA3_|^CUDA|^BICODEC|^VLLM'


Output:
Chunk ID: 43cffc
Wall time: 0.0517 seconds
Process exited with code 1
Original token count: 0
Output:


Input:
*** Begin Patch
*** Update File: scripts/stress_test_optimized_local.py
@@
     def observed_inflight_max(self) -> Optional[int]:
-        values = [r.request_inflight for r in self.successes if isinstance(r.request_inflight, int)]
-        values.extend([r.client_inflight for r in self.successes if isinstance(r.client_inflight, int)])
+        values = [r.request_inflight for r in self.results if isinstance(r.request_inflight, int)]
+        values.extend([r.client_inflight for r in self.results if isinstance(r.client_inflight, int)])
         if not values:
             return None
         return max(values)
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_optimized_local.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


I also fixed a reporting gap: concurrency peak now tracks all requests (including failures), so the summary table will show true client/server inflight peaks even when every request times out. I’ll run one quick single-level check.


python3 -u scripts/stress_test_optimized_local.py --levels 1 --text short --max-tokens 128 --timeout-seconds 8 --no-chunking --output /tmp/bench_inflight_check.json


Output:
Chunk ID: 78fe09
Wall time: 30.0011 seconds
Process running with session ID 46671
Original token count: 26
Output:
  Health: healthy
  Version: LLM
  GPU: True

Warming up (1 request, non-streaming)...
  warmup ok: 0%


Output:
Chunk ID: fee324
Wall time: 30.0009 seconds
Process running with session ID 46671
Original token count: 0
Output:


Output:
Chunk ID: 884e55
Wall time: 22.4342 seconds
Process exited with code 0
Original token count: 302
Output:

========================================================================
  1C-SHORT-NON-STREAMING
  10 requests, 1 concurrent, non-streaming, chunking=False
========================================================================
  Success:     0/10 (0%)
  Wall time:   80.63s
  Throughput:  0.12 req/s
  Audio total: 0.00s
  Eff. RTF:    0.000
  Peak inflight: 1

  GPU:
    Memory: 41871MB avg, 41871MB peak / 81920MB
    Util:   4% avg, 71% peak
    Temp:   28C avg, 30C peak

  Errors:
    [10x] TIMEOUT

==============================================================================================================
  STRESS BENCHMARK SUMMARY
==============================================================================================================
Level                        OK%     RPS     p50     p95  GPU%   GPUmem    RTF Inflight
--------------------------------------------------------------------------------------------------------------
1c-short-non-streaming       0%    0.12     ---     ---    4%   41871MB  0.000        1
==============================================================================================================

Results written to /tmp/bench_inflight_check.json


python3 - <<'PY'
import json
p='/tmp/bench_inflight_check.json'
with open(p) as f:
 d=json.load(f)[0]
print(d['client_inflight_peak'])
print('keys',d['requests'][0].keys())
print('timing',d['requests'][0]['timing'])
print('gpu',d['requests'][0]['gpu_samples'])
PY


Output:
Chunk ID: 4851e7
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 120
Output:
1
keys dict_keys(['request_id', 'success', 'status_code', 'latency_ms', 'ttfb_ms', 'server_ttfb_ms', 'audio_bytes', 'audio_seconds', 'request_inflight', 'client_inflight', 'error', 'timing', 'gpu_samples'])
timing {'client_inflight_start': 1}
gpu {'start': {'gpu_utilization_pct': 0.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 28.0}, 'end': {'gpu_utilization_pct': 0.0, 'memory_used_mb': 41871.0, 'memory_total_mb': 81920.0, 'temperature_c': 28.0}}


Implemented and run. I added/verified detailed per-request logging in the local benchmark runner and executed stress runs against the optimized local path.

What I updated in `scripts/stress_test_optimized_local.py`:
- `RequestResult` now stores `client_inflight` and `gpu_samples`.
- `LevelResult` now stores `client_inflight_peak`.
- `run_level()` now tracks client-side concurrent in-flight count and writes it into each request’s timing as `client_inflight_start`.
- `save_results()` now persists:
  - `client_inflight` / `client_inflight_peak`
  - per-request `gpu_samples` (`start` / `end` snapshot from `nvidia-smi`)
  - all timing keys parsed from response headers.
- parser now also accepts `x-llm-generation-ms`.
- inflight peak reporting now includes both successful and failed requests (so it still reports under timeout/fail regimes).

Benchmark commands run:
- `python3 -u scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 128 --timeout-seconds 8 --output /tmp/bench_short_optimized_valid2.json`
- `python3 -u scripts/stress_test_optimized_local.py --levels 1 --text short --max-tokens 128 --timeout-seconds 8 --no-chunking --output /tmp/bench_short_nochunk1.json`
- `python3 -u scripts/stress_test_optimized_local.py --levels 1 --text short --max-tokens 128 --timeout-seconds 8 --stream --output /tmp/bench_short_stream1.json`
- `python3 -u scripts/stress_test_optimized_local.py --levels 1 --text short --max-tokens 128 --timeout-seconds 8 --no-chunking --output /tmp/bench_inflight_check.json` (to confirm inflight reporting)

Latest results:
- `1C-SHORT-NON-STREAMING` (10 req, timeout 8s): `0/10` success, wall `~80.6s`, throughput `0.12 req/s`, GPU avg util `~4%` (peak up to `71%`), temp `28–30C`.
- `2C-SHORT-NON-STREAMING` (10 req): `0/10` success, wall `~40.6s`, throughput `0.25 req/s`.
- `4C-SHORT-NON-STREAMING` (12 req): `0/12` success, wall `~24.7s`, throughput `0.49 req/s`.
- `1C-SHORT-STREAMING` (10 req): `0/10` success, wall `~80.7s`, throughput `0.12 req/s`.
- no-chunking shows the same timeout outcome pattern.

Per-request logs now present in JSON:
- `timing` on failed requests only contains `client_inflight_start` (no server perf headers on timeout path).
- `gpu_samples` captured for start/end with `gpu_utilization_pct`, `memory_used_mb`, `memory_total_mb`, `temperature_c`.
- `client_inflight_peak` appears and matches requested concurrency in the observed run (`1`, `2`, `4` where applicable).

Key bottleneck signal:
- No request reaches completion in current path, so `x-llm-*` generation timing (`llm_token_total`, batch wall/gpu, decode, parse, etc.) is not available.
- This is a pre-completion stall (likely server-side generation path never yields/returns response); client sees near-constant 8s timeout wall time and no server-side headers.
- CPU-side client concurrency scales as expected, but throughput is bounded by timeout cadence, not output throughput.

If you want, I can do the next pass to isolate the exact server-side stall point (first-batch wait vs decode loop) by enabling per-stage timestamps in the pipeline and re-running at one request first, then ramping concurrency once a single request completes.