Output:
Chunk ID: 2bfe5c
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2481
Output:
            level.results.append(item)
        else:
            level.results.append(
                RequestResult(
                    success=False,
                    status_code=0,
                    latency_ms=0,
                    error=str(item)[:200],
                    stream=stream,
                )
            )

    if gpu_monitor:
        level.gpu_summary = gpu_monitor.summary()

    return level

def print_level_report(level: LevelResult):
    print(f"\n{'=' * 72}")
    print(f"  {level.level_name.upper()}")
    print(f"  {level.total_requests} requests, {level.concurrency} concurrent, {level.mode}, chunking={level.chunking}")
    print(f"{'=' * 72}")

    print(f"  Success:     {len(level.successes)}/{level.total_requests} ({level.success_rate:.0%})")
    print(f"  Wall time:   {level.wall_time_seconds:.2f}s")
    print(f"  Throughput:  {level.throughput_rps:.2f} req/s")
    print(f"  Audio total: {level.audio_duration_total:.2f}s")
    print(f"  Eff. RTF:    {level.effective_rtf:.3f}")

    if level.observed_inflight_max() is not None:
        print(f"  Peak inflight: {level.observed_inflight_max()}")

    latency = level.latency_stats
    if latency:
        print("\n  Latency (ms):")
        print(
            f"    min={latency['min']:.0f} avg={latency['avg']:.0f} "
            f"p50={latency['p50']:.0f} p95={latency['p95']:.0f} p99={latency['p99']:.0f} max={latency['max']:.0f}"
        )

    ttfb = level.ttfb_stats
    if ttfb and not level.mode.startswith("stream"):
        print("\n  Server TTFB (ms):")
        print(
            f"    min={ttfb['min']:.0f} avg={ttfb['avg']:.0f} "
            f"p50={ttfb['p50']:.0f} p95={ttfb['p95']:.0f} max={ttfb['max']:.0f}"
        )

    if level.mode == "non-streaming":
        llm = level.timing_stats("generation_ms")
        if llm:
            print("\n  Generation (ms):")
            print(f"    min={llm['min']:.0f} avg={llm['avg']:.0f} p50={llm['p50']:.0f} p95={llm['p95']:.0f} max={llm['max']:.0f}")
        wall = level.timing_stats("llm_batch_wall_ms_total")
        if wall:
            print("\n  LLM batch wall (ms):")
            print(f"    min={wall['min']:.0f} avg={wall['avg']:.0f} p50={wall['p50']:.0f} p95={wall['p95']:.0f} max={wall['max']:.0f}")
        gpu = level.timing_stats("llm_batch_gpu_ms_total")
        if gpu:
            print("\n  LLM batch GPU (ms):")
            print(f"    min={gpu['min']:.0f} avg={gpu['avg']:.0f} p50={gpu['p50']:.0f} p95={gpu['p95']:.0f} max={gpu['max']:.0f}")
        parse = level.timing_stats("llm_parse_ms")
        if parse:
            print("\n  Parse (ms):")
            print(f"    avg={parse['avg']:.0f} p95={parse['p95']:.0f}")
        decode = level.timing_stats("llm_decode_wall_ms_total")
        if decode:
            print("\n  Decode (ms):")
            print(f"    avg={decode['avg']:.0f} p95={decode['p95']:.0f}")

        token_ms, token_count = level.timing_rate("generation_ms", "llm_token_total")
        if token_count:
            print(f"  Token timing: avg {token_ms / token_count * 1000:.3f} us/token")

    if level.gpu_summary and "memory_mb" in level.gpu_summary:
        gpu_summary = level.gpu_summary
        mem = gpu_summary["memory_mb"]
        util = gpu_summary["gpu_util_pct"]
        temp = gpu_summary["temperature_c"]
        print("\n  GPU:")
        print(f"    Memory: {mem['avg']:.0f}MB avg, {mem['max']:.0f}MB peak / {mem['total']:.0f}MB")
        print(f"    Util:   {util['avg']:.0f}% avg, {util['max']:.0f}% peak")
        print(f"    Temp:   {temp['avg']:.0f}C avg, {temp['max']:.0f}C peak")

    errors = level.error_summary()
    if errors:
        print("\n  Errors:")
        for key, count in sorted(errors.items(), key=lambda kv: -kv[1]):
            print(f"    [{count}x] {key}")

def print_summary_table(levels: List[LevelResult]):
    print(f"\n{'=' * 110}")
    print("  STRESS BENCHMARK SUMMARY")
    print(f"{'=' * 110}")
    header = f"{'Level':<26} {'OK%':>5} {'RPS':>7} {'p50':>7} {'p95':>7} {'GPU%':>5} {'GPUmem':>8} {'RTF':>6} {'Inflight':>8}"
    print(header)
    print("-" * 110)

    for level in levels:
        lat = level.latency_stats
        p50 = f"{lat.get('p50', 0):.0f}" if lat else "---"
        p95 = f"{lat.get('p95', 0):.0f}" if lat else "---"
        gpu_util = level.gpu_summary.get("gpu_util_pct", {}).get("avg", 0)
        gpu_mem = level.gpu_summary.get("memory_mb", {}).get("max", 0)
        inflight = level.observed_inflight_max() or 0
        print(
            f"{level.level_name:<26} "
            f"{level.success_rate:>4.0%} "
            f"{level.throughput_rps:>7.2f} "
            f"{p50:>7} "
            f"{p95:>7} "
            f"{gpu_util:>4.0f}% "
            f"{gpu_mem:>7.0f}MB "
            f"{level.effective_rtf:>6.3f} "
            f"{inflight:>8d}"
        )

    print("=" * 110)

def save_results(levels: List[LevelResult], path: str):
    payload = []
    for level in levels:
        payload.append({
            "level": level.level_name,
            "concurrency": level.concurrency,
            "mode": level.mode,
            "chunking": level.chunking,
            "text_category": level.text_category,
            "success_rate": level.success_rate,
            "throughput_rps": level.throughput_rps,
            "wall_time_s": level.wall_time_seconds,
            "latency_ms": level.latency_stats,
            "ttfb_ms": level.ttfb_stats,
            "effective_rtf": level.effective_rtf,
            "audio_total_seconds": level.audio_duration_total,
            "observed_inflight_max": level.observed_inflight_max(),
            "errors": level.error_summary(),
            "gpu": level.gpu_summary,
            "timing_averages": {
                "generation_ms": level.timing_stats("generation_ms"),
                "llm_batch_wall_ms_total": level.timing_stats("llm_batch_wall_ms_total"),
                "llm_batch_gpu_ms_total": level.timing_stats("llm_batch_gpu_ms_total"),
                "llm_parse_ms": level.timing_stats("llm_parse_ms"),
                "llm_decode_wall_ms_total": level.timing_stats("llm_decode_wall_ms_total"),
            },
            "requests": [
                {
                    "request_id": r.request_id,
                    "success": r.success,
                    "status_code": r.status_code,
                    "latency_ms": r.latency_ms,
                    "ttfb_ms": r.ttfb_ms,
                    "server_ttfb_ms": r.server_ttfb_ms,
                    "audio_bytes": r.audio_bytes,
                    "audio_seconds": r.audio_seconds,
                    "request_inflight": r.request_inflight,
                    "error": r.error,
                    "timing": r.timing,
                }
                for r in level.results
            ],
        })

    with open(path, "w") as fd:
        json.dump(payload, fd, indent=2)

async def run_stress_test(args):
    base_url = args.url

    try:
        async with httpx.AsyncClient() as client:
            response = await client.get(f"{base_url}/v1/tts/health", timeout=10)
            health = response.json()
            print(f"  Health: {health.get('status')}")
            print(f"  Version: {health.get('model_version')}")
            print(f"  GPU: {health.get('gpu_available')}")
    except Exception as e:
        print(f"ERROR: server not reachable at {base_url}: {e}")
        sys.exit(1)

    levels = [int(v) for v in args.levels.split(",") if v.strip()]
    gpu_monitor = GPUMonitor(interval_seconds=0.5)
    gpu_monitor.start()

    mode_label = "streaming" if args.stream else "non-streaming"
    results: List[LevelResult] = []

    print(f"\nWarming up (1 request, non-streaming)...")
    warmup = await run_level(
        base_url=base_url,
        num_requests=1,
        concurrency=1,
        text_category=args.text,
        chunking=args.chunking,
        stream=False,
        max_tokens=args.max_tokens,
        gpu_monitor=gpu_monitor,
    )
    print(f"  warmup ok: {warmup.success_rate:.0%}")

    for conc in levels:
        num_requests = max(conc * 3, 10)
        num_requests = min(num_requests, 200)
        level = await run_level(
            base_url=base_url,
            num_requests=num_requests,
            concurrency=conc,
            text_category=args.text,
            chunking=args.chunking,
            stream=args.stream,
            max_tokens=args.max_tokens,
            gpu_monitor=gpu_monitor,
        )
        print_level_report(level)
        results.append(level)
        if conc != levels[-1]:
            print("  cooldown 3s...")
            await asyncio.sleep(3)

    gpu_monitor.stop()
    print_summary_table(results)

    if args.output:
        save_results(results, args.output)
        print(f"\nResults written to {args.output}")

def main():
    parser = argparse.ArgumentParser(description="Detailed local optimized-path TTS benchmark")
    parser.add_argument("--url", default=DEFAULT_URL, help=f"Server URL (default: {DEFAULT_URL})")
    parser.add_argument("--levels", default="1,5,10,20,50", help="Comma-separated concurrency levels")
    parser.add_argument("--text", default="short", choices=["short", "medium", "long", "mixed"], help="Input text category")
    parser.add_argument("--chunking", action="store_true", help="Enable server-side chunking", default=True)
    parser.add_argument("--no-chunking", action="store_false", dest="chunking", help="Disable chunking")
    parser.add_argument(
        "--max-tokens",
        type=int,
        default=256,
        help="max_tokens passed to /v1/tts/generate (default: 256)",
    )
    parser.add_argument("--stream", action="store_true", help="Use streaming endpoint")
    parser.add_argument("--output", default="stress_test_optimized_local.json", help="Output JSON path")

    args = parser.parse_args()
    asyncio.run(run_stress_test(args))

if __name__ == "__main__":
    main()


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


I’ve confirmed the benchmark script already parses and aggregates many timing fields. Next I’m checking pipeline-level timing emission so we can capture any missing stages (decode iteration, parse, wall vs GPU) and ensure request-level logs correlate with concurrency and token-level cost.


rg -n "timing|llm_gpu_batch|decode|parse|batch|token|return_timing|performance" veena3modal/core/pipeline.py && sed -n '1,320p' veena3modal/core/pipeline.py && sed -n '320,700p' veena3modal/core/pipeline.py && sed -n '700,1100p' veena3modal/core/pipeline.py


Output:
Chunk ID: 8f9b9f
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 5405
Output:
4:End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
31:def _detailed_timing_enabled() -> bool:
45:    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
52:        bicodec_decoder,
60:            bicodec_decoder: BiCodecDecoder instance
64:        self.bicodec_decoder = bicodec_decoder
75:        max_tokens: int = DEFAULT_MAX_TOKENS,
78:        return_timing: bool = False,
93:            max_tokens: Max BiCodec tokens to generate (default: 2048)
95:            return_timing: If true, returns (audio_bytes, metrics_dict)
99:            If return_timing is true, returns tuple(audio_bytes, metrics).
109:            max_tokens=max_tokens,
112:            skip_special_tokens=False,
121:        emit_timing = return_timing or _detailed_timing_enabled()
128:        # Collect all outputs and track generation timing
130:        token_batches: List[int] = []
131:        batch_wall_ms: List[float] = []
132:        batch_gpu_ms: List[float] = []
133:        total_token_ids = 0
134:        first_token_ms: Optional[float] = None
144:                start_event = torch.cuda.Event(enable_timing=True)
145:                end_event = torch.cuda.Event(enable_timing=True)
154:            batch_ms = (wall_end - wall_start) * 1000
155:            batch_wall_ms.append(batch_ms)
157:            batch_gpu_ms_value: Optional[float] = None
161:                batch_gpu_ms_value = start_event.elapsed_time(end_event)
162:            if batch_gpu_ms_value is not None:
163:                batch_gpu_ms.append(batch_gpu_ms_value)
165:            total_generated = len(request_output.outputs[0].token_ids)
166:            new_tokens = max(0, total_generated - total_token_ids)
167:            total_token_ids = total_generated
168:            token_batches.append(new_tokens)
170:            if first_token_ms is None:
171:                first_token_ms = (wall_end - t_start) * 1000
173:            if emit_timing:
175:                    "tts_llm_batch",
179:                        "batch_index": len(token_batches),
180:                        "batch_tokens": new_tokens,
181:                        "llm_batch_wall_ms": round(batch_ms, 3),
182:                        "llm_batch_gpu_ms": round(batch_gpu_ms_value, 3)
183:                        if batch_gpu_ms_value is not None
190:            if emit_timing:
196:            if return_timing:
200:                    "token_total": 0,
201:                    "batch_count": 0,
205:        # Extract generated text (contains BiCodec tokens)
206:        parse_start = asyncio.get_event_loop().time()
208:        generated_ids = final_output.outputs[0].token_ids
210:        # Extract BiCodec tokens using regex
211:        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)
213:        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000
216:            if emit_timing:
218:                    "tts_bicodec_parse_failed",
224:                        "token_total": len(generated_ids),
228:                "❌ No BiCodec tokens found in output",
233:                    "generated_tokens": len(generated_ids),
234:                    "semantic_tokens": len(semantic_ids),
235:                    "global_tokens": len(global_ids),
239:            if return_timing:
242:                    "ttfb_ms": int(first_token_ms or 0),
243:                    "token_total": len(generated_ids),
244:                    "batch_count": len(token_batches),
248:        # Validate tokens
249:        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
250:            logger.error("❌ BiCodec token validation failed", extra={
253:                "semantic_tokens": len(semantic_ids),
254:                "global_tokens": len(global_ids),
256:            if return_timing:
259:                    "ttfb_ms": int(first_token_ms or 0),
260:                    "token_total": len(generated_ids),
261:                    "batch_count": len(token_batches),
266:        decode_start = asyncio.get_event_loop().time()
267:        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
268:        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000
271:            if emit_timing:
272:                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
273:            logger.error("❌ BiCodec decode failed", extra={
277:            if return_timing:
280:                    "ttfb_ms": int(first_token_ms or 0),
281:                    "token_total": len(generated_ids),
282:                    "batch_count": len(token_batches),
288:        timing_metrics: Dict[str, Any] = {
292:            "batch_count": len(token_batches),
293:            "token_total": len(generated_ids),
294:            "token_batches": token_batches,
295:            "ttfb_ms": int(first_token_ms or 0),
296:            "llm_wall_ms": int(sum(batch_wall_ms)),
297:            "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
298:            "llm_gpu_batch_ms_total": int(sum(batch_gpu_ms)),
299:            "llm_gpu_batch_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
300:            "llm_gpu_batch_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
301:            "llm_gpu_batch_ms_p50": int(median(batch_gpu_ms)) if batch_gpu_ms else 0,
302:            "decode_cpu_ms": int(decode_ms),
303:            "parse_ms": int(parse_ms),
306:        if token_batches:
307:            timing_metrics["tokens_per_batch_min"] = min(token_batches)
308:            timing_metrics["tokens_per_batch_max"] = max(token_batches)
309:            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))
311:        if emit_timing:
312:            logger.info("tts_llm_complete", extra=timing_metrics)
317:        if return_timing:
318:            return wav_bytes, timing_metrics
328:        max_tokens: int = DEFAULT_MAX_TOKENS,
331:        return_timing: bool = False,
350:            max_tokens: Max BiCodec tokens to generate
352:            return_timing: If true, returns (audio_bytes, metrics_dict)
356:            If return_timing is true, returns tuple(audio_bytes, metrics).
365:            max_tokens=max_tokens,
368:            return_timing=return_timing,
372:    def _extract_bicodec_tokens(self, generated_text: str) -> Tuple[List[int], List[int]]:
374:        Extract BiCodec semantic and global tokens from generated text using regex.
376:        Spark TTS generates tokens in the format:
381:            generated_text: Generated text containing BiCodec token markers
385:            Returns ([], []) if no tokens found
387:        # Extract semantic tokens
391:        # Extract global tokens
397:            logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")
"""
Spark TTS Generation Pipeline

End-to-end pipeline for TTS generation with BiCodec audio tokenizer.
Migrated from Veena3/Orpheus (SNAC) to Spark TTS (BiCodec).
"""

import asyncio
import logging
import os
import re
from statistics import mean, median
from typing import Optional, List, Dict, Any, Tuple

from vllm import SamplingParams
import torch

logger = logging.getLogger(__name__)

from veena3modal.core.constants import (
    TRAINING_STOP_TOKEN_IDS,
    DEFAULT_TEMPERATURE,
    DEFAULT_TOP_K,
    DEFAULT_TOP_P,
    DEFAULT_MAX_TOKENS,
    DEFAULT_SEED,
    AUDIO_SAMPLE_RATE,
)

def _detailed_timing_enabled() -> bool:
    """
    Return True when benchmark logs are enabled.

    Controlled by environment variable:
      VEENA3_TIMING_LOGS=true|1
    """
    return os.getenv("VEENA3_TIMING_LOGS", "").lower() in {"1", "true", "yes", "on"}

class SparkTTSPipeline:
    """
    End-to-end TTS pipeline for Spark TTS with BiCodec.

    Replaces SNAC-based token extraction with BiCodec regex-based extraction.
    """

    def __init__(
        self,
        model,
        prompt_builder,
        bicodec_decoder,
    ):
        """
        Initialize pipeline.

        Args:
            model: SparkTTS Model instance
            prompt_builder: IndicPromptBuilder instance (with Spark TTS format)
            bicodec_decoder: BiCodecDecoder instance
        """
        self.model = model
        self.prompt_builder = prompt_builder
        self.bicodec_decoder = bicodec_decoder

        print(f"🚀 SparkTTSPipeline initialized")

    async def generate_speech(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ) -> Optional[bytes] | tuple[Optional[bytes], Dict[str, Any]]:
        """
        Generate speech audio (non-streaming) using Spark TTS.

        NOTE: This method signature changed from description-based to speaker-based
        to align with Spark TTS architecture.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize (with optional [emotion] tags)
            temperature: Sampling temperature (default: 0.8 for Spark TTS)
            top_k: Top-k sampling (default: 50)
            top_p: Nucleus sampling (default: 1.0)
            max_tokens: Max BiCodec tokens to generate (default: 2048)
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Build prompt using Spark TTS format
        prompt = self.prompt_builder.build_prefix(speaker, text)

        # Configure sampling for Spark TTS
        sampling_params = SamplingParams(
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            stop=TRAINING_STOP_TOKEN_IDS,
            skip_special_tokens=False,
            seed=seed,
        )

        # Use vLLM engine to generate
        if request_id is None:
            import uuid
            request_id = f"req-{uuid.uuid4().hex[:12]}"

        emit_timing = return_timing or _detailed_timing_enabled()
        results_generator = self.model.engine.generate(
            prompt=prompt,
            sampling_params=sampling_params,
            request_id=request_id,
        )

        # Collect all outputs and track generation timing
        final_output = None
        token_batches: List[int] = []
        batch_wall_ms: List[float] = []
        batch_gpu_ms: List[float] = []
        total_token_ids = 0
        first_token_ms: Optional[float] = None
        t_start = asyncio.get_event_loop().time()

        iterator = results_generator.__aiter__()
        while True:
            wall_start = asyncio.get_event_loop().time()
            start_event = None
            end_event = None
            if torch.cuda.is_available():
                torch.cuda.synchronize()
                start_event = torch.cuda.Event(enable_timing=True)
                end_event = torch.cuda.Event(enable_timing=True)
                start_event.record()

            try:
                request_output = await iterator.__anext__()
            except StopAsyncIteration:
                break

            wall_end = asyncio.get_event_loop().time()
            batch_ms = (wall_end - wall_start) * 1000
            batch_wall_ms.append(batch_ms)

            batch_gpu_ms_value: Optional[float] = None
            if start_event is not None and end_event is not None:
                end_event.record()
                torch.cuda.synchronize()
                batch_gpu_ms_value = start_event.elapsed_time(end_event)
            if batch_gpu_ms_value is not None:
                batch_gpu_ms.append(batch_gpu_ms_value)

            total_generated = len(request_output.outputs[0].token_ids)
            new_tokens = max(0, total_generated - total_token_ids)
            total_token_ids = total_generated
            token_batches.append(new_tokens)

            if first_token_ms is None:
                first_token_ms = (wall_end - t_start) * 1000

            if emit_timing:
                logger.info(
                    "tts_llm_batch",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "batch_index": len(token_batches),
                        "batch_tokens": new_tokens,
                        "llm_batch_wall_ms": round(batch_ms, 3),
                        "llm_batch_gpu_ms": round(batch_gpu_ms_value, 3)
                        if batch_gpu_ms_value is not None
                        else None,
                    },
                )
            final_output = request_output

        if final_output is None:
            if emit_timing:
                logger.warning(
                    "tts_llm_no_output",
                    extra={"speaker": speaker, "request_id": request_id},
                )
            print(f"❌ Generation failed: no output")
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": 0,
                    "token_total": 0,
                    "batch_count": 0,
                }
            return None

        # Extract generated text (contains BiCodec tokens)
        parse_start = asyncio.get_event_loop().time()
        generated_text = final_output.outputs[0].text
        generated_ids = final_output.outputs[0].token_ids

        # Extract BiCodec tokens using regex
        semantic_ids, global_ids = self._extract_bicodec_tokens(generated_text)

        parse_ms = (asyncio.get_event_loop().time() - parse_start) * 1000

        if not semantic_ids or not global_ids:
            if emit_timing:
                logger.error(
                    "tts_bicodec_parse_failed",
                    extra={
                        "speaker": speaker,
                        "request_id": request_id,
                        "semantic_count": len(semantic_ids),
                        "global_count": len(global_ids),
                        "token_total": len(generated_ids),
                    },
                )
            logger.error(
                "❌ No BiCodec tokens found in output",
                extra={
                    "speaker": speaker,
                    "text_length": len(text),
                    "text_preview": text[:100],
                    "generated_tokens": len(generated_ids),
                    "semantic_tokens": len(semantic_ids),
                    "global_tokens": len(global_ids),
                    "generated_text_preview": generated_text[:500],
                }
            )
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Validate tokens
        if not self.bicodec_decoder.validate_tokens(semantic_ids, global_ids):
            logger.error("❌ BiCodec token validation failed", extra={
                "speaker": speaker,
                "text_length": len(text),
                "semantic_tokens": len(semantic_ids),
                "global_tokens": len(global_ids),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        # Decode to audio
        decode_start = asyncio.get_event_loop().time()
        audio_bytes = self.bicodec_decoder.decode_to_bytes(semantic_ids, global_ids)
        decode_ms = (asyncio.get_event_loop().time() - decode_start) * 1000

        if audio_bytes is None:
            if emit_timing:
                logger.error("tts_decode_failed", extra={"speaker": speaker, "request_id": request_id})
            logger.error("❌ BiCodec decode failed", extra={
                "speaker": speaker,
                "text_length": len(text),
            })
            if return_timing:
                return None, {
                    "request_id": request_id,
                    "ttfb_ms": int(first_token_ms or 0),
                    "token_total": len(generated_ids),
                    "batch_count": len(token_batches),
                }
            return None

        total_ms = (asyncio.get_event_loop().time() - t_start) * 1000

        timing_metrics: Dict[str, Any] = {
            "request_id": request_id,
            "speaker": speaker,
            "text_length": len(text),
            "batch_count": len(token_batches),
            "token_total": len(generated_ids),
            "token_batches": token_batches,
            "ttfb_ms": int(first_token_ms or 0),
            "llm_wall_ms": int(sum(batch_wall_ms)),
            "llm_avg_batch_ms": int(mean(batch_wall_ms)) if batch_wall_ms else 0,
            "llm_gpu_batch_ms_total": int(sum(batch_gpu_ms)),
            "llm_gpu_batch_ms_min": int(min(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_max": int(max(batch_gpu_ms)) if batch_gpu_ms else 0,
            "llm_gpu_batch_ms_p50": int(median(batch_gpu_ms)) if batch_gpu_ms else 0,
            "decode_cpu_ms": int(decode_ms),
            "parse_ms": int(parse_ms),
            "total_ms": int(total_ms),
        }
        if token_batches:
            timing_metrics["tokens_per_batch_min"] = min(token_batches)
            timing_metrics["tokens_per_batch_max"] = max(token_batches)
            timing_metrics["tokens_per_batch_p50"] = int(median(token_batches))

        if emit_timing:
            logger.info("tts_llm_complete", extra=timing_metrics)

        from veena3modal.audio.utils import add_wav_header
        wav_bytes = add_wav_header(audio_bytes, sample_rate=AUDIO_SAMPLE_RATE)

        if return_timing:
            return wav_bytes, timing_metrics
        return wav_bytes

    async def generate_speech_indic(
        self,
        speaker: str,
        text: str,
        temperature: float = DEFAULT_TEMPERATURE,
        top_k: int = DEFAULT_TOP_K,
        top_p: float = DEFAULT_TOP_P,
        max_tokens: int = DEFAULT_MAX_TOKENS,
        repetition_penalty: float = 1.05,
        seed: Optional[int] = None,
        return_timing: bool = False,
        request_id: Optional[str] = None,
    ):
        """
        Generate speech audio for Spark TTS (speaker-based).

        This method is kept for backward compatibility with existing API.
        It delegates to generate_speech() with the same implementation.

        Args:
            speaker: Speaker name (one of 12 predefined speakers)
            text: Text to synthesize with inline emotion tags
                Examples:
                - "Hello! Welcome."
                - "Hello <laugh> this is fun!" (will be normalized)
                - "नमस्ते! [excited] आज का दिन बहुत अच्छा है।"
            temperature: Sampling temperature
            top_k: Top-k sampling
            top_p: Nucleus sampling
            max_tokens: Max BiCodec tokens to generate
            seed: Random seed for reproducibility
            return_timing: If true, returns (audio_bytes, metrics_dict)

        Returns:
            Audio bytes (int16 PCM WAV, 16kHz mono) or None if failed.
            If return_timing is true, returns tuple(audio_bytes, metrics).
        """
        # Delegate to generate_speech() with the same implementation for Spark TTS
        return await self.generate_speech(
            speaker=speaker,
            text=text,
            temperature=temperature,
            top_k=top_k,
            top_p=top_p,
            max_tokens=max_tokens,
            repetition_penalty=repetition_penalty,
            seed=seed,
            return_timing=return_timing,
            request_id=request_id,
        )

    def _extract_bicodec_tokens(self, generated_text: str) -> Tuple[List[int], List[int]]:
        """
        Extract BiCodec semantic and global tokens from generated text using regex.

        Spark TTS generates tokens in the format:
        - <|bicodec_semantic_{id}|>
        - <|bicodec_global_{id}|>

        Args:
            generated_text: Generated text containing BiCodec token markers

        Returns:
            Tuple of (semantic_ids, global_ids)
            Returns ([], []) if no tokens found
        """
        # Extract semantic tokens
        semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", generated_text)
        semantic_ids = [int(t) for t in semantic_matches]

        # Extract global tokens
        global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", generated_text)
        global_ids = [int(t) for t in global_matches]

        # Only log if there's an issue
        if not semantic_ids and not global_ids:
            logger.error(f"❌ No BiCodec tokens found! Generated text (first 1000 chars):\n{generated_text[:1000]}")

        return semantic_ids, global_ids


Summary: auto


rg -n "def main\(|local_server|uvicorn|FastAPI|__name__ == '__main__'|run\(|app =" veena3modal/app.py veena3modal/local_server.py veena3modal -g '!**/*.pyc'


Output:
Chunk ID: 7cd1d8
Wall time: 0.0502 seconds
Process exited with code 0
Original token count: 2009
Output:
veena3modal/app.py:45:        # FastAPI
veena3modal/app.py:47:        "uvicorn[standard]>=0.23.0",
veena3modal/app.py:132:app = modal.App(
veena3modal/app.py:244:        Return FastAPI ASGI app for serving requests.
veena3modal/app.py:361:def main():
veena3modal/local_server.py:2:Local entry point for Veena3 TTS - runs the same FastAPI app without Modal.
veena3modal/local_server.py:8:4. Serves the FastAPI app via uvicorn
veena3modal/local_server.py:12:    python -m veena3modal.local_server
veena3modal/local_server.py:15:    python -m veena3modal.local_server --model-path /path/to/spark_tts_4speaker
veena3modal/local_server.py:18:    python -m veena3modal.local_server --gpu-memory 0.5 --port 8080
veena3modal/local_server.py:21:    python -m veena3modal.local_server --enable-sr --sr-path /path/to/ap_bwe/16kto48k
veena3modal/local_server.py:24:    python -m veena3modal.local_server --device cpu
veena3modal/local_server.py:151:        "     python -m veena3modal.local_server --model-path /path/to/model"
veena3modal/local_server.py:246:    Create the FastAPI app for local serving.
veena3modal/local_server.py:261:  python -m veena3modal.local_server                          # Auto-download model, serve on :8000
veena3modal/local_server.py:262:  python -m veena3modal.local_server --port 8080              # Custom port
veena3modal/local_server.py:263:  python -m veena3modal.local_server --model-path ./my_model  # Use local model
veena3modal/local_server.py:264:  python -m veena3modal.local_server --gpu-memory 0.5         # Limit GPU memory
veena3modal/local_server.py:265:  python -m veena3modal.local_server --workers 2              # Multiple workers (careful: each loads model)
veena3modal/local_server.py:350:        help="Number of uvicorn workers (default: 1, each loads full model)",
veena3modal/local_server.py:381:def main():
veena3modal/local_server.py:389:    4. Create FastAPI app
veena3modal/local_server.py:390:    5. Run uvicorn
veena3modal/local_server.py:436:    # Run uvicorn
veena3modal/local_server.py:437:    # NOTE: We use the factory string so uvicorn can find the app.
veena3modal/local_server.py:438:    # With --reload, uvicorn reimports, but the runtime singleton persists in memory.
veena3modal/local_server.py:439:    import uvicorn
veena3modal/local_server.py:451:    uvicorn.run(
veena3modal/local_server.py:452:        "veena3modal.local_server:create_local_app",
veena3modal/local_server.py:2:Local entry point for Veena3 TTS - runs the same FastAPI app without Modal.
veena3modal/local_server.py:8:4. Serves the FastAPI app via uvicorn
veena3modal/local_server.py:12:    python -m veena3modal.local_server
veena3modal/local_server.py:15:    python -m veena3modal.local_server --model-path /path/to/spark_tts_4speaker
veena3modal/local_server.py:18:    python -m veena3modal.local_server --gpu-memory 0.5 --port 8080
veena3modal/local_server.py:21:    python -m veena3modal.local_server --enable-sr --sr-path /path/to/ap_bwe/16kto48k
veena3modal/local_server.py:24:    python -m veena3modal.local_server --device cpu
veena3modal/local_server.py:151:        "     python -m veena3modal.local_server --model-path /path/to/model"
veena3modal/local_server.py:246:    Create the FastAPI app for local serving.
veena3modal/local_server.py:261:  python -m veena3modal.local_server                          # Auto-download model, serve on :8000
veena3modal/local_server.py:262:  python -m veena3modal.local_server --port 8080              # Custom port
veena3modal/local_server.py:263:  python -m veena3modal.local_server --model-path ./my_model  # Use local model
veena3modal/local_server.py:264:  python -m veena3modal.local_server --gpu-memory 0.5         # Limit GPU memory
veena3modal/local_server.py:265:  python -m veena3modal.local_server --workers 2              # Multiple workers (careful: each loads model)
veena3modal/local_server.py:350:        help="Number of uvicorn workers (default: 1, each loads full model)",
veena3modal/local_server.py:381:def main():
veena3modal/local_server.py:389:    4. Create FastAPI app
veena3modal/local_server.py:390:    5. Run uvicorn
veena3modal/local_server.py:436:    # Run uvicorn
veena3modal/local_server.py:437:    # NOTE: We use the factory string so uvicorn can find the app.
veena3modal/local_server.py:438:    # With --reload, uvicorn reimports, but the runtime singleton persists in memory.
veena3modal/local_server.py:439:    import uvicorn
veena3modal/local_server.py:451:    uvicorn.run(
veena3modal/local_server.py:452:        "veena3modal.local_server:create_local_app",
veena3modal/app.py:45:        # FastAPI
veena3modal/app.py:47:        "uvicorn[standard]>=0.23.0",
veena3modal/app.py:132:app = modal.App(
veena3modal/app.py:244:        Return FastAPI ASGI app for serving requests.
veena3modal/app.py:361:def main():
veena3modal/services/tts_runtime.py:259:        # Update FastAPI app with model version
veena3modal/services/tts_runtime.py:352:                pool.submit(lambda: asyncio.run(_run_all())).result()
veena3modal/services/tts_runtime.py:357:        asyncio.run(_run_all())
veena3modal/api/fastapi_app.py:2:FastAPI ASGI app factory for Modal deployment.
veena3modal/api/fastapi_app.py:11:# to ensure FastAPI recognizes Request type annotations properly
veena3modal/api/fastapi_app.py:220:    Factory function to create FastAPI app.
veena3modal/api/fastapi_app.py:222:    Import inside factory so local tooling can import this module without FastAPI installed.
veena3modal/api/fastapi_app.py:224:    from fastapi import FastAPI, HTTPException, Request
veena3modal/api/fastapi_app.py:230:    app = FastAPI(
veena3modal/api/__init__.py:1:"""API layer for the Modal-native Veena3 service (FastAPI)."""
veena3modal/api/websocket_handler.py:215:    This is the main handler called from FastAPI WebSocket route.
veena3modal/api/websocket_handler.py:218:        websocket: FastAPI WebSocket instance
veena3modal/api/websocket_handler.py:528:    Add WebSocket routes to FastAPI app.
veena3modal/api/websocket_handler.py:533:        app: FastAPI application instance
veena3modal/__main__.py:1:"""Allow running as: python -m veena3modal (shortcut for local_server)."""
veena3modal/__main__.py:2:from veena3modal.local_server import main
veena3modal/audio/encoder.py:206:        process = subprocess.run(
veena3modal/audio/encoder.py:267:        process = subprocess.run(
veena3modal/audio/encoder.py:321:        process = subprocess.run(
veena3modal/audio/encoder.py:382:        process = subprocess.run(
veena3modal/tests/integration/test_tts_generate.py:65:    app = create_app()
veena3modal/tests/edge_cases/test_concurrent_access.py:170:        results = asyncio.run(run_tasks())
veena3modal/tests/modal_live/test_deployed_endpoint.py:319:        results = asyncio.run(run_concurrent(3))
veena3modal/tests/modal_live/test_load.py:270:        result = asyncio.run(run_load_test(
veena3modal/tests/modal_live/test_load.py:288:        result = asyncio.run(run_load_test(
veena3modal/tests/modal_live/test_load.py:308:        result = asyncio.run(run_load_test(
veena3modal/tests/modal_live/test_load.py:327:        result = asyncio.run(run_load_test(
veena3modal/tests/modal_live/test_load.py:345:        result = asyncio.run(run_load_test(
veena3modal/tests/modal_live/test_load.py:416:    asyncio.run(run_full_load_test_suite())
veena3modal/tests/modal_live/debug_asr_issue.py:329:def main():
veena3modal/tests/unit/test_fastapi_app.py:2:Unit tests for veena3modal FastAPI app.
veena3modal/tests/unit/test_fastapi_app.py:14:        """Ensure create_app() returns a FastAPI application."""
veena3modal/tests/unit/test_fastapi_app.py:17:        app = create_app()
veena3modal/tests/unit/test_fastapi_app.py:19:        # Check it's a FastAPI app (has routes, title, version)
veena3modal/tests/unit/test_fastapi_app.py:28:        app = create_app()
veena3modal/tests/unit/test_fastapi_app.py:40:        """Create a test client for the FastAPI app."""
veena3modal/tests/unit/test_fastapi_app.py:44:        app = create_app()


I found a local server entry point already in `veena3modal/local_server.py`. I’m validating it plus the benchmark CLI so we can run repeatable stress runs and, if needed, add new probes specifically where the request path stalls.


sed -n '220,520p' veena3modal/local_server.py


Output:
Chunk ID: 127f72
Wall time: 0.0505 seconds
Process exited with code 0
Original token count: 2011
Output:
    logger.info(f"  Engines:     {num_engines}")
    logger.info(f"  SR:          {'enabled' if enable_sr else 'disabled'}")
    logger.info("=" * 60)

    runtime = initialize_runtime(
        model_path=llm_path,
        bicodec_path=bicodec_path,
        sr_checkpoint_dir=sr_checkpoint_dir if enable_sr else None,
        device=device,
        hf_token=hf_token,
        gpu_memory_utilization=gpu_memory_utilization,
        enable_sr=enable_sr,
        num_engines=num_engines,
    )

    elapsed = time.time() - start
    logger.info(f"Runtime initialized in {elapsed:.1f}s")
    logger.info(f"  Model version: {runtime.model_version}")
    logger.info(f"  Streaming: {'available' if runtime.streaming_pipeline else 'unavailable'}")
    logger.info(f"  Super-resolution: {'loaded' if runtime.sr_service and runtime.sr_service.is_loaded else 'disabled'}")

    return runtime

def create_local_app():
    """
    Create the FastAPI app for local serving.

    Same app factory as Modal deployment, but called from local context.
    """
    from veena3modal.api.fastapi_app import create_app
    return create_app()

def parse_args():
    """Parse CLI arguments for local server configuration."""
    parser = argparse.ArgumentParser(
        description="Veena3 TTS Local Server - GPU-accelerated text-to-speech",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python -m veena3modal.local_server                          # Auto-download model, serve on :8000
  python -m veena3modal.local_server --port 8080              # Custom port
  python -m veena3modal.local_server --model-path ./my_model  # Use local model
  python -m veena3modal.local_server --gpu-memory 0.5         # Limit GPU memory
  python -m veena3modal.local_server --workers 2              # Multiple workers (careful: each loads model)
        """,
    )

    # Model configuration
    model_group = parser.add_argument_group("Model")
    model_group.add_argument(
        "--model-path",
        type=str,
        default=os.environ.get("MODEL_PATH", DEFAULT_LOCAL_MODEL_DIR),
        help=f"Path to Spark TTS model (default: {DEFAULT_LOCAL_MODEL_DIR})",
    )
    model_group.add_argument(
        "--hf-repo",
        type=str,
        default=PRIVATE_HF_REPO,
        help=f"HuggingFace repo to download from (default: {PRIVATE_HF_REPO})",
    )
    model_group.add_argument(
        "--hf-token",
        type=str,
        default=os.environ.get("HF_TOKEN"),
        help="HuggingFace token (default: from HF_TOKEN env var)",
    )
    model_group.add_argument(
        "--skip-download",
        action="store_true",
        help="Don't auto-download model (fail if not found)",
    )

    # GPU configuration
    gpu_group = parser.add_argument_group("GPU")
    gpu_group.add_argument(
        "--device",
        type=str,
        default="cuda",
        choices=["cuda", "cpu"],
        help="Device for inference (default: cuda)",
    )
    gpu_group.add_argument(
        "--gpu-memory",
        type=float,
        default=float(os.environ.get("GPU_MEMORY_UTILIZATION", "0.25")),
        help="vLLM GPU memory utilization fraction (default: 0.25)",
    )
    gpu_group.add_argument(
        "--num-engines",
        type=int,
        default=int(os.environ.get("NUM_ENGINES", "1")),
        help="Number of vLLM engine instances (default: 1, use 2-3 for multi-engine mode)",
    )

    # Super-resolution
    sr_group = parser.add_argument_group("Super-Resolution")
    sr_group.add_argument(
        "--enable-sr",
        action="store_true",
        default=False,
        help="Enable AP-BWE super-resolution (16kHz -> 48kHz)",
    )
    sr_group.add_argument(
        "--sr-path",
        type=str,
        default=os.environ.get("AP_BWE_CHECKPOINT_DIR"),
        help="Path to AP-BWE checkpoint directory",
    )

    # Server configuration
    server_group = parser.add_argument_group("Server")
    server_group.add_argument(
        "--host",
        type=str,
        default="0.0.0.0",
        help="Bind address (default: 0.0.0.0)",
    )
    server_group.add_argument(
        "--port",
        type=int,
        default=int(os.environ.get("PORT", "8000")),
        help="Port to serve on (default: 8000)",
    )
    server_group.add_argument(
        "--workers",
        type=int,
        default=1,
        help="Number of uvicorn workers (default: 1, each loads full model)",
    )
    server_group.add_argument(
        "--reload",
        action="store_true",
        default=False,
        help="Enable auto-reload for development (incompatible with --workers > 1)",
    )

    # Auth
    auth_group = parser.add_argument_group("Auth")
    auth_group.add_argument(
        "--auth",
        action="store_true",
        default=False,
        help="Enable API key authentication (default: disabled for local)",
    )

    # Logging
    log_group = parser.add_argument_group("Logging")
    log_group.add_argument(
        "--log-level",
        type=str,
        default="INFO",
        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
        help="Log level (default: INFO)",
    )

    return parser.parse_args()

def main():
    """
    Main entry point for local TTS server.

    Lifecycle:
    1. Parse args + configure logging
    2. Download model if needed
    3. Initialize TTS runtime (vLLM engine, decoders, pipelines)
    4. Create FastAPI app
    5. Run uvicorn
    """
    args = parse_args()

    # Configure logging
    logging.basicConfig(
        level=getattr(logging, args.log_level),
        format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )

    # Auth bypass for local development (unless --auth flag is set)
    if not args.auth:
        os.environ["AUTH_BYPASS_MODE"] = "true"
        logger.info("Auth bypass enabled (local mode). Use --auth to require API keys.")
    else:
        os.environ.setdefault("AUTH_BYPASS_MODE", "false")

    # Resolve model path
    # Spark TTS root has config.yaml; LLM subdir has config.json; fine-tuned may have config.json at root
    model_path = args.model_path
    model_exists = any(
        os.path.exists(os.path.join(model_path, marker))
        for marker in ("config.json", "config.yaml", "LLM/config.json")
    )
    if not model_exists:
        if args.skip_download:
            logger.error(f"Model not found at {model_path} and --skip-download is set")
            sys.exit(1)
        model_path = download_model(
            hf_repo=args.hf_repo,
            local_dir=model_path,
            hf_token=args.hf_token,
        )

    # Initialize TTS runtime (this is the expensive step - loads model onto GPU)
    initialize_local_runtime(
        model_path=model_path,
        device=args.device,
        gpu_memory_utilization=args.gpu_memory,
        enable_sr=args.enable_sr,
        sr_checkpoint_dir=args.sr_path,
        hf_token=args.hf_token,
        num_engines=args.num_engines,
    )

    # Run uvicorn
    # NOTE: We use the factory string so uvicorn can find the app.
    # With --reload, uvicorn reimports, but the runtime singleton persists in memory.
    import uvicorn

    logger.info("")
    logger.info("=" * 60)
    logger.info(f"Veena3 TTS Local Server starting")
    logger.info(f"  URL: http://{args.host}:{args.port}")
    logger.info(f"  Health: http://{args.host}:{args.port}/v1/tts/health")
    logger.info(f"  Generate: POST http://{args.host}:{args.port}/v1/tts/generate")
    logger.info(f"  Metrics: http://{args.host}:{args.port}/v1/tts/metrics")
    logger.info(f"  Docs: http://{args.host}:{args.port}/docs")
    logger.info("=" * 60)

    uvicorn.run(
        "veena3modal.local_server:create_local_app",
        factory=True,
        host=args.host,
        port=args.port,
        workers=args.workers,
        reload=args.reload,
        log_level=args.log_level.lower(),
        # Performance: keep-alive for connection reuse under load
        timeout_keep_alive=30,
    )

if __name__ == "__main__":
    main()


I’m going to quickly validate whether a local server is currently active and whether the non-streaming path returns for a single request; this determines whether we should debug first or run full stress immediately.


curl -sS http://localhost:8000/v1/tts/health | head -c 500 && echo


Output:
Chunk ID: 76b406
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 33
Output:
{"status":"healthy","model_loaded":true,"model_version":"LLM","uptime_seconds":208.28,"gpu_available":true,"app_version":"0.1.0"}


time curl -sS -m 20 -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"hello","speaker":"lipakshi","chunking":false,"max_tokens":64,"format":"wav"}' -D - -o /tmp/out.wav; echo status:$?; ls -l /tmp/out.wav


Output:
Chunk ID: 1083ff
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 74
Output:
HTTP/1.1 400 Bad Request
date: Tue, 17 Feb 2026 00:29:05 GMT
server: uvicorn
x-request-id: 846c8388-d13d-4c2d-8351-9b434503224c
content-length: 350
content-type: application/json

status:0

real	0m0.007s
user	0m0.002s
sys	0m0.004s
-rw-rw-r-- 1 ubuntu ubuntu 350 Feb 17 00:29 /tmp/out.wav


I confirmed the endpoint responds, but got a 400 input validation. I’ll send the exact schema fields next to confirm generation can run end-to-end and then proceed with timed benchmarks.


curl -sS -m 20 -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"hello","speaker":"lipakshi","max_tokens":64,"format":"wav","chunking":true,"stream":false}' -i


Output:
Chunk ID: 2c38f6
Wall time: 0.0517 seconds
Process exited with code 0
Original token count: 135
Output:
HTTP/1.1 400 Bad Request
date: Tue, 17 Feb 2026 00:29:07 GMT
server: uvicorn
x-request-id: b1896c4d-23ca-4020-a824-5a9cb9302401
content-length: 350
content-type: application/json

{"error":{"code":"VALIDATION_ERROR","message":"1 validation error for TTSGenerateRequest\nmax_tokens\n  Input should be greater than or equal to 128 [type=greater_than_equal, input_value=64, input_type=int]\n    For further information visit https://errors.pydantic.dev/2.12/v/greater_than_equal","request_id":"b1896c4d-23ca-4020-a824-5a9cb9302401"}}


curl -sS -m 45 -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"hello","speaker":"lipakshi","max_tokens":128,"format":"wav","chunking":true,"stream":false}' -i


Output:
Chunk ID: 641bf5
Wall time: 10.0029 seconds
Process running with session ID 62315
Original token count: 0
Output:


Output:
Chunk ID: 0cabdf
Wall time: 30.0009 seconds
Process running with session ID 62315
Original token count: 0
Output:


Output:
Chunk ID: 881a94
Wall time: 1.8519 seconds
Process exited with code 28
Original token count: 20
Output:
curl: (28) Operation timed out after 45001 milliseconds with 0 bytes received


The single-request generation call is still hanging at 128 tokens, so I’m checking metrics counters and then adding generation-level timeout/instrumentation in the pipeline so we can capture exactly where the request blocks.


curl -sS http://localhost:8000/v1/tts/metrics | rg -n "veena3_tts_|requests|completed|inflight|tokens|fail"


Output:
Chunk ID: d25d5e
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 437
Output:
1:# HELP veena3_tts_requests_total Total TTS requests received
2:# TYPE veena3_tts_requests_total counter
3:veena3_tts_requests_total{format="wav",speaker="lipakshi",stream="false"} 3.0
4:veena3_tts_requests_total{format="AudioFormat_WAV",speaker="lipakshi",stream="false"} 2.0
5:veena3_tts_requests_total{format="wav",speaker="vardan",stream="false"} 1.0
6:# HELP veena3_tts_requests_created Total TTS requests received
7:# TYPE veena3_tts_requests_created gauge
8:veena3_tts_requests_created{format="wav",speaker="lipakshi",stream="false"} 1.7712879407518544e+09
9:veena3_tts_requests_created{format="AudioFormat_WAV",speaker="lipakshi",stream="false"} 1.7712880434559853e+09
10:veena3_tts_requests_created{format="wav",speaker="vardan",stream="false"} 1.7712881808878982e+09
11:# HELP veena3_tts_requests_completed_total Total TTS requests completed successfully
12:# TYPE veena3_tts_requests_completed_total counter
13:# HELP veena3_tts_requests_failed_total Total TTS requests failed
14:# TYPE veena3_tts_requests_failed_total counter
15:# HELP veena3_tts_ttfb_seconds Time to first byte in seconds
16:# TYPE veena3_tts_ttfb_seconds histogram
17:# HELP veena3_tts_rtf Real-time factor (generation_time / audio_duration)
18:# TYPE veena3_tts_rtf histogram
19:# HELP veena3_tts_audio_duration_seconds Generated audio duration in seconds
20:# TYPE veena3_tts_audio_duration_seconds histogram
21:# HELP veena3_tts_chunks_sent Number of audio chunks sent in streaming mode
22:# TYPE veena3_tts_chunks_sent histogram
23:# HELP veena3_tts_model_load_seconds Time taken to load the model
24:# TYPE veena3_tts_model_load_seconds gauge
25:# HELP veena3_tts_model_loaded Whether the model is loaded (1) or not (0)
26:# TYPE veena3_tts_model_loaded gauge


ps -ef | rg "veena3modal.local_server|uvicorn" | head -n 20


Output:
Chunk ID: 1c65c1
Wall time: 0.0506 seconds
Process exited with code 0
Original token count: 111
Output:
ubuntu   3706410 3675569 28 00:24 ?        00:01:24 python3 -m veena3modal.local_server --skip-download --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --port 8000 --gpu-memory 0.15 --num-engines 1 --log-level INFO
ubuntu   3710218 3675569  0 00:29 ?        00:00:00 /bin/bash -lc ps -ef | rg "veena3modal.local_server|uvicorn" | head -n 20
ubuntu   3710223 3710218  0 00:29 ?        00:00:00 rg veena3modal.local_server|uvicorn


Output:
Chunk ID: 08d4c6
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 1670
Output:
total 0
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 0 -> /dev/null
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 1 -> pipe:[330579671]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 10 -> anon_inode:[eventfd]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 11 -> anon_inode:[eventpoll]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 12 -> anon_inode:[eventfd]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 13 -> anon_inode:[eventpoll]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 14 -> anon_inode:[eventfd]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 15 -> anon_inode:[eventpoll]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 16 -> anon_inode:[eventfd]
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 17 -> /dev/urandom
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 18 -> pipe:[330591268]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 19 -> socket:[330579716]
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 2 -> pipe:[330579672]
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 20 -> /dev/urandom
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 21 -> /home/ubuntu/.cursor-server/data/logs/20260216T232129/remoteagent.log
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 22 -> anon_inode:[eventfd]
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 23 -> pipe:[330591269]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 24 -> socket:[330579717]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 25 -> socket:[330586279]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 26 -> socket:[330565434]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 27 -> anon_inode:[eventfd]
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 28 -> pipe:[330582553]
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 29 -> /home/ubuntu/.cursor-server/data/logs/20260216T232129/exthost2/remoteExtHostTelemetry.log
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 3 -> /dev/nvidiactl
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 30 -> /home/ubuntu/.cursor-server/data/logs/20260216T232129/exthost2/remoteexthost.log
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 31 -> pipe:[330582553]
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 32 -> pipe:[330582554]
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 33 -> pipe:[330582554]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 34 -> /dev/nvidiactl
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 35 -> /home/ubuntu/.cursor-server/data/logs/20260216T232129/exthost2/vscode.github/GitHub.log
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 36 -> /dev/nvidia-uvm
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 37 -> /home/ubuntu/.cursor-server/data/logs/20260216T232129/exthost2/anysphere.cursor-agent-exec/Mcp FileSystem Writer.log
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 38 -> /dev/nvidia0
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 39 -> /home/ubuntu/.cursor-server/data/logs/20260216T232129/exthost2/vscode.git/Git.log
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 4 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 40 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 41 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 42 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 43 -> socket:[330582562]
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 44 -> /home/ubuntu/.cursor-server/data/logs/20260216T232129/exthost2/output_logging_20260216T232936/1-Cursor Agent.log
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 45 -> /home/ubuntu/.cursor-server/data/logs/20260216T232129/exthost2/anysphere.cursor-mcp/MCP Logs.log
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 46 -> /home/ubuntu/.cursor-server/data/logs/20260216T232129/exthost2/anysphere.cursor-mcp/MCP user-Vast.ai Documentation –.log
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 47 -> /home/ubuntu/.cursor-server/data/logs/20260216T232129/exthost2/anysphere.cursor-agent-exec/Cursor Agent Exec.log
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 48 -> anon_inode:[eventfd]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 49 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 5 -> /dev/nvidia-uvm
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 50 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 51 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 52 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 53 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 54 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 55 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 56 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 57 -> anon_inode:[eventfd]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 58 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 59 -> /dev/nvidia0
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 6 -> /dev/nvidia-caps/nvidia-cap2
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 60 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 61 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 62 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 63 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 64 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 65 -> /dev/nvidia0
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 66 -> pipe:[330582565]
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 67 -> pipe:[330582565]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 68 -> anon_inode:[eventpoll]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 69 -> socket:[330582566]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 7 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 70 -> socket:[330582567]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 71 -> anon_inode:[eventpoll]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 72 -> anon_inode:[io_uring]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 73 -> anon_inode:[io_uring]
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 74 -> pipe:[330565437]
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 75 -> pipe:[330565437]
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 76 -> pipe:[330565438]
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 77 -> pipe:[330565438]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 78 -> anon_inode:[eventfd]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 79 -> socket:[330565439]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 8 -> /dev/nvidia0
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 80 -> socket:[330565440]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 81 -> socket:[330565441]
lr-x------ 1 ubuntu ubuntu 64 Feb 17 00:25 82 -> /dev/null
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 83 -> socket:[330602349]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:27 84 -> socket:[330582957]
lrwx------ 1 ubuntu ubuntu 64 Feb 17 00:25 9 -> anon_inode:[eventfd]
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 /proc/3706410/fd/1 -> pipe:[330579671]
l-wx------ 1 ubuntu ubuntu 64 Feb 17 00:25 /proc/3706410/fd/2 -> pipe:[330579672]
/proc/3706410/fd/1: symbolic link to pipe:[330579671]
/proc/3706410/fd/2: symbolic link to pipe:[330579672]


rg -n "TRAINING_STOP_TOKEN_IDS|DEFAULT_MAX_TOKENS|DEFAULT_TEMPERATURE|TOP_P|TOP_K" veena3modal/core/constants.py && sed -n '1,220p' veena3modal/core/constants.py


Output:
Chunk ID: ea52f1
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1559
Output:
12:TRAINING_STOP_TOKEN_IDS = ["<|im_end|>"]  # String-based stop token for Spark TTS
127:DEFAULT_TEMPERATURE = 0.8  # Higher than old model (was 0.4)
128:DEFAULT_TOP_K = 50
129:DEFAULT_TOP_P = 1.0
130:DEFAULT_MAX_TOKENS = 4096  # Increased to handle complex multilingual generation
"""
Spark TTS Constants

Token IDs, special tokens, speakers, and emotions for Spark TTS model.
Migrated from Veena3/Orpheus to Spark TTS (BiCodec) architecture.
"""

import os

# Stop tokens for generation
# Spark TTS uses <|im_end|> as stop token
TRAINING_STOP_TOKEN_IDS = ["<|im_end|>"]  # String-based stop token for Spark TTS

# Spark TTS Speaker System - 12 Speakers from Training
# Maps user-facing speaker names to internal speaker IDs (speaker_0 to speaker_11)
# Model: BayAreaBoys/spark_tts_4speaker (HuggingFace) has 12 speaker tokens
SPEAKER_MAP = {
    "lipakshi": 0,    # speaker_0
    "vardan": 1,      # speaker_1
    "reet": 2,        # speaker_2
    "Nandini": 3,     # speaker_3
    "krishna": 4,     # speaker_4
    "anika": 5,       # speaker_5
    "adarsh": 6,      # speaker_6
    "Nilay": 7,       # speaker_7
    "Aarvi": 8,       # speaker_8
    "Asha": 9,        # speaker_9
    "Bittu": 10,      # speaker_10
    "Mira": 11,       # speaker_11
}

# Friendly speaker name mappings (user-facing → internal)
# Users can use these friendly names instead of internal names
FRIENDLY_SPEAKER_MAP = {
    "Mitra": "lipakshi",
    "Aaranya": "reet",
    "Taru": "Nandini",
    "Neer": "Nilay",
    "Dhruva": "vardan",
    "Ira": "anika",
    "Veda": "adarsh",
    "Aria": "krishna",
    # Direct mappings for new speakers
    "Aarvi": "Aarvi",
    "Asha": "Asha",
    "Bittu": "Bittu",
    "Mira": "Mira",
}

# Valid speaker names (for API validation)
# Include both internal names and friendly names
INDIC_SPEAKERS = list(SPEAKER_MAP.keys())
ALL_SPEAKER_NAMES = INDIC_SPEAKERS + list(FRIENDLY_SPEAKER_MAP.keys())

def resolve_speaker_name(name: str) -> str:
    """
    Resolve friendly speaker name to internal name.

    Args:
        name: Speaker name (friendly or internal)

    Returns:
        Internal speaker name

    Examples:
        resolve_speaker_name("Mitra") -> "lipakshi"
        resolve_speaker_name("lipakshi") -> "lipakshi"
    """
    # Check if it's a friendly name
    if name in FRIENDLY_SPEAKER_MAP:
        return FRIENDLY_SPEAKER_MAP[name]

    # Check if it's already an internal name
    if name in INDIC_SPEAKERS:
        return name

    # Invalid speaker name
    raise ValueError(f"Invalid speaker name: {name}. Valid names: {', '.join(ALL_SPEAKER_NAMES)}")

# Spark TTS Emotion Tags (bracket format: [emotion])
# NOTE: API users must now use [emotion] instead of <emotion>
INDIC_EMOTION_TAGS = [
    "[angry]",
    "[curious]",
    "[excited]",
    "[giggle]",
    "[laughs harder]",
    "[laughs]",
    "[screams]",
    "[sighs]",
    "[sings]",
    "[whispers]"
]

# Legacy emotion tags for backward compatibility mapping
# Maps old <emotion> format to new [emotion] format
LEGACY_EMOTION_MAP = {
    "<angry>": "[angry]",
    "<curious>": "[curious]",
    "<excited>": "[excited]",
    "<giggle>": "[giggle]",
    "<laugh_harder>": "[laughs harder]",
    "<laugh>": "[laughs]",
    "<scream>": "[screams]",
    "<sigh>": "[sighs]",
    "<sing>": "[sings]",
    "<whisper>": "[whispers]"
}

# All emotion tags (for validation)
ALL_EMOTION_TAGS = INDIC_EMOTION_TAGS

# Model configuration
DEFAULT_MODEL_PATH = "BayAreaBoys/spark_tts_4speaker"
DEFAULT_MAX_MODEL_LEN = 4096  # Lower than old model (was 8192)

# BiCodec Audio Tokenizer
# Location: resolved from env var MODEL_PATH or default
BICODEC_TOKENIZER_PATH = os.environ.get('MODEL_PATH', os.environ.get('SPARK_TTS_MODEL_PATH', '/models/spark_tts_4speaker'))

# Audio configuration
AUDIO_SAMPLE_RATE = 16000  # Changed from 24kHz to 16kHz for BiCodec
AUDIO_CHANNELS = 1
AUDIO_BITS_PER_SAMPLE = 16

# Generation defaults for Spark TTS
DEFAULT_TEMPERATURE = 0.8  # Higher than old model (was 0.4)
DEFAULT_TOP_K = 50
DEFAULT_TOP_P = 1.0
DEFAULT_MAX_TOKENS = 4096  # Increased to handle complex multilingual generation
DEFAULT_MIN_TOKENS = 28  # Minimum tokens for generation (legacy compatibility)
DEFAULT_REPETITION_PENALTY = 1.0  # Repetition penalty (legacy compatibility)
DEFAULT_SEED = None  # None = random, set integer for reproducibility

# vLLM Configuration optimizations for Spark TTS
# NOTE: gpu_memory_utilization adjusted based on available memory
# Production: 0.85, Limited memory: 0.3-0.5
#
# OPTIMIZATION Dec 2025:
# - enable_chunked_prefill: Prevents long prompts from blocking concurrent streams
# - async_scheduling: DISABLED - caused EngineCore issues in production
# OPTIMIZATION Feb 2026:
# Reduced gpu_memory_utilization from 0.85 to 0.25
# Model is only 0.5B (~1.3GB weights). At 0.85, vLLM pre-allocated 65GB for KV cache
# (enough for 1,399 concurrent seqs) when peak actual need is ~100 concurrent.
# At 0.25: ~18GB KV cache, supports ~380 concurrent seqs (4x peak), frees ~48GB VRAM.
# This enables fitting on L4 (24GB) or even T4 (16GB) GPUs.
VLLM_CONFIG = {
    "dtype": "bfloat16",
    "gpu_memory_utilization": 0.25,  # 0.5B model needs <2GB; 0.25 gives ~380 concurrent seqs
    "max_model_len": 4096,
    "max_num_batched_tokens": 4096,
    "enable_prefix_caching": True,  # Cache common prompts for faster TTFB
    "enable_chunked_prefill": True,  # Chunk long prefills to avoid blocking concurrent streams
    "enforce_eager": False,  # Enable CUDA graphs for low latency
    "disable_log_stats": False,
    "trust_remote_code": True,
    "tensor_parallel_size": 1,
}

# Streaming configuration (may need adjustment for BiCodec)
STREAM_BUFFER_SIZE = 50  # Buffer size for streaming (BiCodec-specific, TBD)
BICODEC_BATCH_SIZE = 64
BICODEC_BATCH_TIMEOUT_MS = 15

# Backward compatibility: Old SNAC constants for files not yet migrated
# These are NOT used by Spark TTS but needed by legacy streaming code
# TODO: Remove once streaming_pipeline.py is fully migrated to BiCodec
CODE_START_TOKEN_ID = 128257  # Legacy SNAC constant
CODE_END_TOKEN_ID = 128258    # Legacy SNAC constant
SNAC_MIN_ID = 128266          # Legacy SNAC constant
SNAC_MAX_ID = 156937          # Legacy SNAC constant


sed -n '1,320p' veena3modal/core/model_loader.py


Output:
Chunk ID: 211368
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 1800
Output:
"""
Spark TTS Model Loader

Loads Spark TTS model with vLLM engine and validates speakers/emotions.
Migrated from Veena3/Orpheus to Spark TTS architecture.
"""

import os
import torch
from typing import Optional, Dict, Any
from transformers import AutoTokenizer
from vllm import AsyncLLMEngine, AsyncEngineArgs, SamplingParams

from veena3modal.core.constants import (
    INDIC_EMOTION_TAGS,
    INDIC_SPEAKERS,
    SPEAKER_MAP,
    DEFAULT_MODEL_PATH,
    DEFAULT_MAX_MODEL_LEN,
    VLLM_CONFIG,
)

class SparkTTSModel:
    """
    Spark TTS Model with vLLM inference engine and BiCodec audio tokenizer.

    Model: BayAreaBoys/spark_tts_4speaker
    Architecture: Qwen2ForCausalLM with BiCodec audio tokenization
    Speakers: 12 predefined (mapped to speaker_0-11 as per training)
    Emotions: 10 tags in [bracket] format

    CRITICAL REQUIREMENTS:
    - Load tokenizer with special Spark TTS tokens
    - Configure vLLM with optimizations (prefix caching, CUDA graphs)
    - Support speaker mapping (lipakshi → speaker_0, etc.)
    """

    def __init__(
        self,
        model_path: str = None,
        model_type: str = "indic_speakers",  # Default to indic_speakers for backward compatibility
        hf_token: str = None,
        **engine_kwargs
    ):
        """
        Initialize Spark TTS model with vLLM.

        Args:
            model_path: Path to model directory (local or HF). If None, uses env var or default.
            model_type: Model type (kept for compatibility, always uses indic_speakers)
            hf_token: HuggingFace token for private models
            **engine_kwargs: Additional vLLM engine arguments (override defaults from VLLM_CONFIG)
        """
        self.model_type = "indic_speakers"  # Spark TTS only supports speaker-based

        # Use provided path, or check environment variable, or use default
        if model_path is None:
            model_path = os.environ.get(
                'SPARK_TTS_MODEL_PATH',
                DEFAULT_MODEL_PATH
            )

        self.model_path = model_path
        self.hf_token = hf_token

        print(f"🚀 Initializing Spark TTS Model")
        print(f"📁 Model path: {model_path}")
        print("🗣  Speakers: 12 predefined "
              "(lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, "
              "Nilay, Aarvi, Asha, Bittu, Mira)")
        print(f"🎭 Emotions: 10 tags in [bracket] format")

        # Load tokenizer
        print(f"\n📝 Loading tokenizer from model...")
        tokenizer_kwargs = {
            'trust_remote_code': True,
        }

        # Add HF token for private repos
        if hf_token:
            tokenizer_kwargs['token'] = hf_token

        # Add emotion tags as special tokens
        special_tokens = {
            'additional_special_tokens': INDIC_EMOTION_TAGS
        }

        self.tokenizer = AutoTokenizer.from_pretrained(
            model_path,
            **tokenizer_kwargs
        )

        # Add special tokens
        self.tokenizer.add_special_tokens(special_tokens)

        print(f"✅ Tokenizer loaded: {len(self.tokenizer)} tokens")

        # Validate speakers and emotions
        self._validate_speakers()
        self._validate_emotion_tags()

        # Initialize vLLM engine with Spark TTS optimizations
        print(f"\n🔧 Initializing vLLM engine with optimizations...")

        # Start with default VLLM_CONFIG and override with kwargs
        engine_kwargs_dict = VLLM_CONFIG.copy()
        engine_kwargs_dict.update({
            'model': model_path,
            'tokenizer': model_path,
        })

        # Add HF token for private models
        if hf_token:
            os.environ['HUGGING_FACE_HUB_TOKEN'] = hf_token

        # Override with any user-provided kwargs
        engine_kwargs_dict.update(engine_kwargs)

        # Print configuration
        print(f"  - dtype: {engine_kwargs_dict['dtype']}")
        print(f"  - max_model_len: {engine_kwargs_dict['max_model_len']}")
        print(f"  - gpu_memory_utilization: {engine_kwargs_dict['gpu_memory_utilization']}")
        print(f"  - enable_prefix_caching: {engine_kwargs_dict.get('enable_prefix_caching', False)}")
        print(f"  - enforce_eager (disable CUDA graphs): {engine_kwargs_dict.get('enforce_eager', True)}")

        engine_args = AsyncEngineArgs(**engine_kwargs_dict)
        self.engine = AsyncLLMEngine.from_engine_args(engine_args)

        print(f"✅ vLLM engine initialized successfully!")
        print(f"\n🎉 Spark TTS Model ready for inference!\n")

    def _validate_emotion_tags(self):
        """
        Validate that emotion tags are properly configured for Spark TTS.

        For Spark TTS, emotion tags use [bracket] format and should be
        added as special tokens to the tokenizer.
        """
        print(f"\n🔍 Validating {len(INDIC_EMOTION_TAGS)} emotion tags for Spark TTS...")

        # Check if emotion tags can be tokenized
        for tag in INDIC_EMOTION_TAGS:
            token_ids = self.tokenizer.encode(tag, add_special_tokens=False)
            # Spark TTS emotion tags may be multiple tokens, that's okay
            if not token_ids:
                print(f"⚠  Warning: Emotion tag {tag} produced no tokens")

        print(f"✅ All {len(INDIC_EMOTION_TAGS)} emotion tags validated")

        # Print sample emotion tags
        print(f"\n📋 Sample emotion tags:")
        for i, tag in enumerate(INDIC_EMOTION_TAGS[:5]):
            token_ids = self.tokenizer.encode(tag, add_special_tokens=False)
            print(f"  {tag}: {len(token_ids)} tokens")
        if len(INDIC_EMOTION_TAGS) > 5:
            print(f"  ...")

    def _validate_speakers(self):
        """
        Validate that all speakers are properly mapped for Spark TTS.

        Spark TTS uses speaker tokens like <|speaker_0|> to <|speaker_7|>.
        User-facing names (lipakshi, vardan, etc.) are mapped to these IDs.
        """
        print(f"\n🔍 Validating {len(INDIC_SPEAKERS)} speakers for Spark TTS...")

        # Validate speaker mapping
        for speaker_name in INDIC_SPEAKERS:
            if speaker_name not in SPEAKER_MAP:
                print(f"⚠  Warning: Speaker {speaker_name} not in SPEAKER_MAP")
                continue

            speaker_id = SPEAKER_MAP[speaker_name]
            speaker_token = f'<|speaker_{speaker_id}|>'
            token_ids = self.tokenizer.encode(speaker_token, add_special_tokens=False)
            print(f"  {speaker_name} → {speaker_token}: {len(token_ids)} tokens")

        print(f"✅ All {len(INDIC_SPEAKERS)} speakers validated with mapping")

    def get_model_type(self) -> str:
        """Get the model type (always 'indic_speakers' for Spark TTS)."""
        return self.model_type

    def get_tokenizer(self):
        """Get the tokenizer."""
        return self.tokenizer

    def get_engine(self):
        """Get the vLLM engine."""
        return self.engine


rg -n "class TTSGenerateRequest|max_tokens|min|max" veena3modal/api/schemas.py && sed -n '1,260p' veena3modal/api/schemas.py


Output:
Chunk ID: 6bbdf8
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 2016
Output:
86:    'whisper': '[whispers]', 'screaming': '[screams]', 'scream': '[screams]',
154:class TTSGenerateRequest(BaseModel):
164:        min_length=1,
165:        max_length=MAX_TEXT_LENGTH,
166:        description="Text to synthesize (max 50,000 characters)"
172:        max_length=50,
179:        max_length=2000,
190:        max_length=50,
231:    max_tokens: int = Field(
375:    """Response metadata for non-streaming /v1/tts/generate."""
"""
Pydantic schemas for TTS API requests/responses.

Ported from veena3srv/apps/api/serializers.py with full validation parity.
No Django dependencies - pure Pydantic for Modal deployment.
"""

from __future__ import annotations

import re
from enum import Enum
from typing import Any, Dict, List, Optional, Union
from uuid import UUID

from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator

# === Constants (mirrored from veena3srv/apps/inference/constants.py) ===
# These are duplicated here to avoid Django import chain; must stay in sync.

MAX_TEXT_LENGTH = 50000

# Spark TTS Speaker System - 12 Speakers
# Model: BayAreaBoys/spark_tts_4speaker (HuggingFace) has 12 speaker tokens
SPEAKER_MAP = {
    "lipakshi": 0, "vardan": 1, "reet": 2, "Nandini": 3,
    "krishna": 4, "anika": 5, "adarsh": 6, "Nilay": 7,
    "Aarvi": 8, "Asha": 9, "Bittu": 10, "Mira": 11,
}

# Friendly speaker name mappings (user-facing → internal)
FRIENDLY_SPEAKER_MAP = {
    "Mitra": "lipakshi", "Aaranya": "reet", "Taru": "Nandini",
    "Neer": "Nilay", "Dhruva": "vardan", "Ira": "anika",
    "Veda": "adarsh", "Aria": "krishna",
    "Aarvi": "Aarvi", "Asha": "Asha", "Bittu": "Bittu", "Mira": "Mira",
}

INDIC_SPEAKERS = list(SPEAKER_MAP.keys())
ALL_SPEAKER_NAMES = INDIC_SPEAKERS + list(FRIENDLY_SPEAKER_MAP.keys())

# Emotion tags (Spark TTS bracket format)
INDIC_EMOTION_TAGS = [
    "[angry]", "[curious]", "[excited]", "[giggle]", "[laughs harder]",
    "[laughs]", "[screams]", "[sighs]", "[sings]", "[whispers]"
]

# Legacy emotion tag mapping (<angle> → [bracket])
LEGACY_EMOTION_MAP = {
    "<angry>": "[angry]", "<curious>": "[curious]", "<excited>": "[excited]",
    "<giggle>": "[giggle]", "<laugh_harder>": "[laughs harder]",
    "<laugh>": "[laughs]", "<scream>": "[screams]", "<sigh>": "[sighs]",
    "<sing>": "[sings]", "<whisper>": "[whispers]"
}

# === Helper Functions ===

def resolve_speaker_name(name: str) -> str:
    """
    Resolve friendly speaker name to internal name.

    Examples:
        resolve_speaker_name("Mitra") -> "lipakshi"
        resolve_speaker_name("lipakshi") -> "lipakshi"

    Raises:
        ValueError: if name is not a valid speaker name
    """
    if name in FRIENDLY_SPEAKER_MAP:
        return FRIENDLY_SPEAKER_MAP[name]
    if name in INDIC_SPEAKERS:
        return name
    raise ValueError(
        f"Invalid speaker name: {name}. Valid names: {', '.join(ALL_SPEAKER_NAMES)}"
    )

# Emotion normalization mapping (various formats → [emotion])
_EMOTION_TO_TAG = {
    **{k.strip('<>'): v for k, v in LEGACY_EMOTION_MAP.items()},
    'laughing': '[laughs]', 'laugh': '[laughs]', 'laughs harder': '[laughs harder]',
    'laugh harder': '[laughs harder]', 'laugh_harder': '[laughs harder]',
    'sighs': '[sighs]', 'sigh': '[sighs]', 'giggles': '[giggle]', 'giggle': '[giggle]',
    'angry': '[angry]', 'excited': '[excited]', 'whispers': '[whispers]',
    'whisper': '[whispers]', 'screaming': '[screams]', 'scream': '[screams]',
    'screams': '[screams]', 'singing': '[sings]', 'sing': '[sings]',
    'sings': '[sings]', 'curious': '[curious]',
}

def normalize_emotion_tags(text: str) -> str:
    """
    Normalize emotion tags to Spark TTS bracket format.

    Conversions:
    - <laugh> → [laughs]
    - [laughing] → [laughs]
    - etc.
    """
    # Convert old <emotion> format to [emotion]
    angle_pattern = r'<([a-z_]+)>'

    def replace_angle(match):
        emotion = match.group(1).lower().strip()
        if emotion in _EMOTION_TO_TAG:
            return _EMOTION_TO_TAG[emotion]
        old_tag = f'<{emotion}>'
        if old_tag in LEGACY_EMOTION_MAP:
            return LEGACY_EMOTION_MAP[old_tag]
        return f'[{emotion}]'  # Unknown, convert to bracket anyway

    text = re.sub(angle_pattern, replace_angle, text)

    # Normalize [emotion] variants
    bracket_pattern = r'\[([^\]]+)\]'

    def replace_bracket(match):
        emotion = match.group(1).lower().strip()
        bracket_tag = f'[{emotion}]'
        if bracket_tag in INDIC_EMOTION_TAGS:
            return bracket_tag
        if emotion in _EMOTION_TO_TAG:
            return _EMOTION_TO_TAG[emotion]
        # Try singular/plural
        if emotion.endswith('s') and emotion[:-1] in _EMOTION_TO_TAG:
            return _EMOTION_TO_TAG[emotion[:-1]]
        if (emotion + 's') in _EMOTION_TO_TAG:
            return _EMOTION_TO_TAG[emotion + 's']
        return bracket_tag  # Keep as-is

    return re.sub(bracket_pattern, replace_bracket, text)

# === Enums ===

class AudioFormat(str, Enum):
    """Supported audio output formats."""
    WAV = "wav"
    OPUS = "opus"
    MP3 = "mp3"
    MULAW = "mulaw"
    FLAC = "flac"

class OutputSampleRate(str, Enum):
    """Output sample rate options."""
    SR_16KHZ = "16khz"
    SR_48KHZ = "48khz"

# === Request Schema ===

class TTSGenerateRequest(BaseModel):
    """
    Request schema for /v1/tts/generate.

    Mirrors validation rules from veena3srv/apps/api/serializers.py.
    """

    # Required
    text: str = Field(
        ...,
        min_length=1,
        max_length=MAX_TEXT_LENGTH,
        description="Text to synthesize (max 50,000 characters)"
    )

    # Speaker (required for indic_speakers model)
    speaker: Optional[str] = Field(
        None,
        max_length=50,
        description="Speaker name (internal or friendly)"
    )

    # Voice design (not currently supported)
    description: Optional[str] = Field(
        None,
        max_length=2000,
        description="Voice description (for voiceDesign model, NOT currently supported)"
    )
    voice_id: Optional[UUID] = Field(
        None,
        description="UUID of pre-created voice profile (for voiceDesign model)"
    )

    # Generation options
    model: Optional[str] = Field(
        None,
        max_length=50,
        description="Model to use (optional). Default: indic_speakers"
    )
    seed: Optional[int] = Field(
        None,
        ge=0,
        le=2**31 - 1,
        description="Random seed for reproducibility (0-2147483647)"
    )
    stream: bool = Field(
        False,
        description="Stream audio as it's generated"
    )
    format: AudioFormat = Field(
        AudioFormat.WAV,
        description="Audio output format"
    )
    sample_rate: int = Field(
        16000,
        description="Sample rate in Hz (default: 16000)"
    )

    # Advanced parameters
    temperature: float = Field(
        0.4,
        ge=0.0,
        le=2.0,
        description="Sampling temperature (0.0-2.0)"
    )
    top_k: int = Field(
        50,
        ge=1,
        le=100,
        description="Top-k sampling (1-100)"
    )
    top_p: float = Field(
        1.0,
        ge=0.0,
        le=1.0,
        description="Nucleus sampling (0.0-1.0)"
    )
    max_tokens: int = Field(
        4096,
        ge=128,
        le=4096,
        description="Maximum BiCodec tokens to generate"
    )
    repetition_penalty: float = Field(
        1.05,
        ge=1.0,
        le=2.0,
        description="Repetition penalty (1.0-2.0)"
    )

    # Preprocessing toggles
    normalize: bool = Field(
        True,
        description="Apply text normalization before TTS"
    )
    normalize_verbose: bool = Field(
        False,
        description="Return normalized text in X-Normalized-Text header"
    )
    chunking: bool = Field(
        True,
        description="Enable intelligent text chunking for long inputs"
    )

    # Output options
    output: OutputSampleRate = Field(
        OutputSampleRate.SR_16KHZ,


I found the endpoint is still hanging at generation time, so I’m going to restart the local server in our control with higher visibility, add a tighter generation timeout + batch-level timestamp logs, then run constrained stress levels to capture per-request GPU/LLM breakdown.


kill 3706410 && sleep 1 && ps -ef | rg 'veena3modal.local_server|python3 -m veena3modal.local_server'


Output:
Chunk ID: 3b58d7
Wall time: 0.9211 seconds
Process exited with code 0
Original token count: 129
Output:
ubuntu   3706410 3675569 27 00:24 ?        00:01:24 python3 -m veena3modal.local_server --skip-download --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --port 8000 --gpu-memory 0.15 --num-engines 1 --log-level INFO
ubuntu   3710373 3675569  0 00:30 ?        00:00:00 /bin/bash -lc kill 3706410 && sleep 1 && ps -ef | rg 'veena3modal.local_server|python3 -m veena3modal.local_server'
ubuntu   3710388 3710373  0 00:30 ?        00:00:00 rg veena3modal.local_server|python3 -m veena3modal.local_server


Summary: auto


kill -9 3706410 && sleep 1 && ps -ef | rg 'veena3modal.local_server|python3 -m veena3modal.local_server'


Output:
Chunk ID: f1c91d
Wall time: 0.9215 seconds
Process exited with code 0
Original token count: 72
Output:
ubuntu   3710409 3675569  0 00:30 ?        00:00:00 /bin/bash -lc kill -9 3706410 && sleep 1 && ps -ef | rg 'veena3modal.local_server|python3 -m veena3modal.local_server'
ubuntu   3710428 3710409  0 00:30 ?        00:00:00 rg veena3modal.local_server|python3 -m veena3modal.local_server


cd /home/ubuntu/veenaModal && NUMBA_DISABLE_JIT=1 PYTHONUNBUFFERED=1 VEENA3_TIMING_LOGS=true python3 -m veena3modal.local_server --skip-download --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --port 8000 --gpu-memory 0.15 --num-engines 1 --log-level INFO


Output:
Chunk ID: ebf4e8
Wall time: 1.0006 seconds
Process running with session ID 63481
Original token count: 458
Output:
2026-02-17 00:30:11 | INFO     | veena3.local | Auth bypass enabled (local mode). Use --auth to require API keys.
2026-02-17 00:30:11 | INFO     | veena3.local | Resolved model paths:
2026-02-17 00:30:11 | INFO     | veena3.local |   LLM (vLLM):    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:30:11 | INFO     | veena3.local |   BiCodec:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:30:11 | INFO     | veena3.local | ============================================================
2026-02-17 00:30:11 | INFO     | veena3.local | Initializing Veena3 TTS Runtime (local)
2026-02-17 00:30:11 | INFO     | veena3.local |   Model:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:30:11 | INFO     | veena3.local |   LLM path:    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:30:11 | INFO     | veena3.local |   BiCodec:     /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:30:11 | INFO     | veena3.local |   Device:      cuda
2026-02-17 00:30:11 | INFO     | veena3.local |   GPU Memory:  15% (total)
2026-02-17 00:30:11 | INFO     | veena3.local |   Engines:     1
2026-02-17 00:30:11 | INFO     | veena3.local |   SR:          disabled
2026-02-17 00:30:11 | INFO     | veena3.local | ============================================================
2026-02-17 00:30:11 | INFO     | veena3modal.services.tts_runtime | Initializing TTS runtime...
2026-02-17 00:30:11 | INFO     | veena3modal.services.tts_runtime |   Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 00:30:11 | INFO     | veena3modal.services.tts_runtime |   BiCodec path: /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 00:30:11 | INFO     | veena3modal.services.tts_runtime |   SR enabled: False, path: None


Output:
Chunk ID: d8d1b8
Wall time: 5.0006 seconds
Process running with session ID 63481
Original token count: 1295
Output:
2026-02-17 00:30:17 | INFO     | veena3modal.services.tts_runtime | Loading SparkTTS model with vLLM...
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.15
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
INFO 02-17 00:30:17 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 00:30:17 [model.py:1561] Using max model len 4096
INFO 02-17 00:30:17 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 00:30:17 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:18 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}


Output:
Chunk ID: 1ffc35
Wall time: 5.0011 seconds
Process running with session ID 63481
Original token count: 440
Output:
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:20 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:60265 backend=nccl
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:20 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:21 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:22 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=3710572) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=3710572) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=3710572)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=3710572)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.39s/it]
(EngineCore_DP0 pid=3710572)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.39s/it]
(EngineCore_DP0 pid=3710572)
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:23 [default_loader.py:291] Loading weights took 1.47 seconds
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:24 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.996470 seconds


Output:
Chunk ID: 76d3f1
Wall time: 5.0001 seconds
Process running with session ID 63481
Original token count: 421
Output:
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:29 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:29 [backends.py:872] Dynamo bytecode transform time: 4.23 s
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:32 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.676 s
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:32 [monitor.py:34] torch.compile takes 4.91 s in total
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:33 [gpu_worker.py:356] Available KV cache memory: 10.13 GiB
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:33 [kv_cache_utils.py:1307] GPU KV cache size: 885,184 tokens
(EngineCore_DP0 pid=3710572) INFO 02-17 00:30:33 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 216.11x
(EngineCore_DP0 pid=3710572)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%| | 0/35 [00:00<?,
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%| | 4/35 [00:00<00:
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|▏| 8/35 [00:00<00:
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|▎| 12/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|▍| 16/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|▌| 20/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|▋| 24/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|▊| 28/35 [00:00<00
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|▉| 32/35 [00:00<00