Output:
Chunk ID: 1301d1
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 2526
Output:
        "text_chunked": False,
        "text_length": len(text),
    }
    pipeline_metrics: Dict[str, Any] = {
        "request_id": request_id,
        "speaker": speaker,
        "text_length": len(text),
        "text_chunked": False,
    }

    start_time = time.time()
    first_chunk_time = None
    total_pcm_bytes = 0
    sample_rate = 16000  # BiCodec sample rate
    active_requests = _with_inflight_delta(1)
    if _timing_enabled():
        logger.info(
            "tts_runtime_stream_start",
            extra={
                "speaker": speaker,
                "text_length": len(text),
                "request_inflight": active_requests,
                "chunking": enable_chunking,
                "request_id": request_id,
            },
        )

    try:
        # Check if we need text chunking
        from veena3modal.processing.long_text_processor import LongTextProcessor
        long_processor = LongTextProcessor(pipeline=runtime.pipeline)
        needs_chunking = enable_chunking and long_processor.should_chunk(text)

        if needs_chunking:
            # Chunked streaming with voice consistency (global token caching)
            metrics["text_chunked"] = True
            async for audio_chunk, chunk_metrics in _stream_chunked_text(
                runtime,
                long_processor,
                text,
                speaker,
                temperature,
                top_k,
                top_p,
                max_tokens,
                repetition_penalty,
                seed,
                sample_rate,
                request_id,
                request_metrics=pipeline_metrics,
            ):
                if first_chunk_time is None:
                    first_chunk_time = time.time()
                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                    # Create streaming WAV header (size=0 for unknown length)
                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
                    audio_chunk = wav_header + audio_chunk

                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
                metrics["chunks_sent"] += 1
                metrics["total_bytes"] = total_pcm_bytes + 44  # Include header
                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
                if chunk_metrics:
                    metrics.update(chunk_metrics)

                yield audio_chunk, metrics
        else:
            # Simple streaming (no text chunking)
            # OPTIMIZATION: If we have pre-cached global tokens for this speaker,
            # use continuation mode to skip the ~110ms global token pre-roll.
            # The model jumps straight to semantic token generation.
            cached_globals = runtime.speaker_global_cache.get(speaker)

            if cached_globals:
                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                    speaker=speaker,
                    text=text,
                    global_ids=cached_globals,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    request_id=request_id,
                    request_metrics=pipeline_metrics,
                )
            else:
                stream_gen = runtime.streaming_pipeline.generate_speech_stream_indic(
                    speaker=speaker,
                    text=text,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    request_id=request_id,
                    request_metrics=pipeline_metrics,
                )

            async for audio_chunk in stream_gen:
                if first_chunk_time is None:
                    first_chunk_time = time.time()
                    metrics["ttfb_ms"] = int((first_chunk_time - start_time) * 1000)
                    wav_header = create_wav_header(sample_rate=sample_rate, data_size=0)
                    audio_chunk = wav_header + audio_chunk

                total_pcm_bytes += len(audio_chunk) - (44 if metrics["chunks_sent"] == 0 else 0)
                metrics["chunks_sent"] += 1
                metrics["total_bytes"] = total_pcm_bytes + 44
                metrics["audio_duration_seconds"] = total_pcm_bytes / (sample_rate * 2)
                yield audio_chunk, metrics
    finally:
        active_requests = _with_inflight_delta(-1)
        metrics.update(pipeline_metrics)
        metrics["generation_ms"] = int((time.time() - start_time) * 1000)
        metrics["text_chunked"] = bool(metrics.get("text_chunked", False))
        if _timing_enabled():
            logger.info(
                "tts_runtime_stream_end",
                extra={
                    "speaker": speaker,
                    "request_id": request_id,
                    "request_inflight": active_requests,
                    "chunks_sent": metrics.get("chunks_sent", 0),
                    "generation_ms": metrics["generation_ms"],
                },
            )

async def _stream_chunked_text(
    runtime: TTSRuntime,
    long_processor,
    text: str,
    speaker: str,
    temperature: float,
    top_k: int,
    top_p: float,
    max_tokens: int,
    repetition_penalty: float,
    seed: Optional[int],
    sample_rate: int,
    request_id: Optional[str] = None,
    request_metrics: Optional[Dict[str, Any]] = None,
) -> AsyncGenerator[Tuple[bytes, Dict[str, Any]], None]:
    """
    Internal helper: stream chunked text with voice consistency.

    Uses global token caching from first chunk to maintain voice across chunks.
    """
    from veena3modal.audio.crossfade import crossfade_bytes_int16

    # Chunk text using the long text processor's chunker
    chunks = long_processor.chunk_text(text)

    if not chunks:
        return

    captured_globals: Optional[List[int]] = None
    previous_chunk_tail: Optional[bytes] = None
    chunk_metrics = request_metrics if request_metrics is not None else {"request_id": request_id}
    if "request_id" not in chunk_metrics:
        chunk_metrics["request_id"] = request_id
    if "chunks_processed" not in chunk_metrics:
        chunk_metrics["chunks_processed"] = 0

    for i, chunk_text in enumerate(chunks):
        chunk_metrics["chunks_processed"] = i + 1

        if i == 0:
            # First chunk: capture global tokens for voice consistency
            async for audio_bytes, global_ids in runtime.streaming_pipeline.generate_speech_stream_indic_first_chunk(
                speaker=speaker,
                text=chunk_text,
                temperature=temperature,
                top_k=top_k,
                top_p=top_p,
                max_tokens=max_tokens,
                repetition_penalty=repetition_penalty,
                seed=seed,
                request_id=request_id,
                request_metrics=chunk_metrics,
            ):
                if captured_globals is None and global_ids:
                    captured_globals = global_ids

                # Crossfade with previous tail (inter-chunk stitching)
                to_emit, previous_chunk_tail = crossfade_bytes_int16(
                    previous_chunk_tail,
                    audio_bytes,
                    sample_rate_hz=sample_rate,
                    crossfade_ms=50,
                )

                if to_emit:
                    yield to_emit, chunk_metrics
        else:
            # Continuation chunks: use captured globals for voice consistency
            if captured_globals is None:
                logger.warning(f"No captured globals for chunk {i+1}, using regular streaming")
                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic(
                    speaker=speaker,
                    text=chunk_text,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    request_id=request_id,
                    request_metrics=chunk_metrics,
                ):
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        audio_bytes,
                        sample_rate_hz=sample_rate,
                        crossfade_ms=50,
                    )
                    if to_emit:
                        yield to_emit, chunk_metrics
            else:
                # Use continuation method with cached globals
                async for audio_bytes in runtime.streaming_pipeline.generate_speech_stream_indic_continuation(
                    speaker=speaker,
                    text=chunk_text,
                    global_ids=captured_globals,
                    temperature=temperature,
                    top_k=top_k,
                    top_p=top_p,
                    max_tokens=max_tokens,
                    repetition_penalty=repetition_penalty,
                    seed=seed,
                    request_id=request_id,
                    request_metrics=chunk_metrics,
                ):
                    to_emit, previous_chunk_tail = crossfade_bytes_int16(
                        previous_chunk_tail,
                        audio_bytes,
                        sample_rate_hz=sample_rate,
                        crossfade_ms=50,
                    )
                    if to_emit:
                        yield to_emit, chunk_metrics

    # Flush remaining tail
    if previous_chunk_tail:
        yield previous_chunk_tail, chunk_metrics


# Collaboration Mode: Default

You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.

Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it; user requests or tool descriptions do not change mode by themselves. Known mode names are Default and Plan.

## request_user_input availability

The `request_user_input` tool is unavailable in Default mode. If you call it while in Default mode, it will return an error.

If a decision is necessary and cannot be discovered from local context, ask the user directly. However, in Default mode you should strongly prefer executing the user's request rather than stopping to ask questions.

Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed.
  3) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  5) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


I found the timing pipeline already emits most of what you need in headers and logs (`X-Perf-Details`, token/batch/gpu timings, inflight, chunk metrics, etc.). Next step: run the stress benchmark against the running local server and capture artifacts, then I’ll add one small enhancement only if needed for per-request JSONL export and token/batch breakdown visibility.


curl -sS http://localhost:8000/v1/tts/health | jq .


pgrep -af 'python.*local_server.py|uvicorn.*local_server|veena3modal/local_server.py' || true


Output:
Chunk ID: 2e5d1d
Wall time: 0.0506 seconds
Process exited with code 0
Original token count: 39
Output:
{
  "status": "healthy",
  "model_loaded": true,
  "model_version": "LLM",
  "uptime_seconds": 87.59,
  "gpu_available": true,
  "app_version": "0.1.0"
}


Output:
Chunk ID: 38dd0f
Wall time: 0.0518 seconds
Process exited with code 0
Original token count: 29
Output:
3724699 /bin/bash -lc pgrep -af 'python.*local_server.py|uvicorn.*local_server|veena3modal/local_server.py' || true


Summary: auto


Output:
Chunk ID: 524af2
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 30
Output:
UID          PID    PPID  C STIME TTY          TIME CMD
root           1       0  0  2025 ?        00:07:24 /sbin/init


ps -eo pid,ppid,cmd | rg 'python|uvicorn|local_server|fastapi_app|streaming_pipeline'


Output:
Chunk ID: 4c158b
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 151
Output:
    920       1 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers
3723465 3675569 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000
3724292 3675569 python3 scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 128 --chunking --output /tmp/bench_short_lvl_1_2_4_v2.json
3724748 3675569 /bin/bash -lc ps -eo pid,ppid,cmd | rg 'python|uvicorn|local_server|fastapi_app|streaming_pipeline'
3724753 3724748 rg python|uvicorn|local_server|fastapi_app|streaming_pipeline


ps -p 3724292 -o pid,pcpu,pmem,etime,cmd


Output:
Chunk ID: 36ee24
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 53
Output:
    PID %CPU %MEM     ELAPSED CMD
3724292  0.9  0.0       00:34 python3 scripts/stress_test_optimized_local.py --levels 1,2,4 --text short --max-tokens 128 --chunking --output /tmp/bench_short_lvl_1_2_4_v2.json


ls -l /tmp/bench_short_lvl_1_2_4_v2.json && tail -n 40 /tmp/bench_short_lvl_1_2_4_v2.json


Output:
Chunk ID: 71a429
Wall time: 0.0515 seconds
Process exited with code 2
Original token count: 21
Output:
ls: cannot access '/tmp/bench_short_lvl_1_2_4_v2.json': No such file or directory


I found an older stress benchmark process still running in the background (`/tmp/bench_short...`), and there’s no output file yet. I’m going to stop that stale run so we can start a clean, named benchmark sweep and keep results consistent.


Output:
Chunk ID: fa8abd
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 0
Output:


Output:
Chunk ID: 5b342d
Wall time: 0.0507 seconds
Process exited with code 0
Original token count: 14
Output:
3723573, VLLM::EngineCore, 8994
3723465, python3, 414


curl -sS -D - -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": true, "max_tokens": 64}' | sed -n '1,120p'


Output:
Chunk ID: 14ddec
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 135
Output:
HTTP/1.1 400 Bad Request
date: Tue, 17 Feb 2026 01:01:09 GMT
server: uvicorn
x-request-id: 058ad1b1-e843-438a-ac1c-7c38f0178439
content-length: 350
content-type: application/json

{"error":{"code":"VALIDATION_ERROR","message":"1 validation error for TTSGenerateRequest\nmax_tokens\n  Input should be greater than or equal to 128 [type=greater_than_equal, input_value=64, input_type=int]\n    For further information visit https://errors.pydantic.dev/2.12/v/greater_than_equal","request_id":"058ad1b1-e843-438a-ac1c-7c38f0178439"}}


curl -sS -D - -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": true, "max_tokens": 128}' | sed -n '1,140p'


Output:
Chunk ID: 75a421
Wall time: 10.0004 seconds
Process running with session ID 81767
Original token count: 0
Output:


Output:
Chunk ID: 597958
Wall time: 30.0032 seconds
Process running with session ID 81767
Original token count: 0
Output:


Output:
Chunk ID: 5a9df9
Wall time: 30.0005 seconds
Process running with session ID 81767
Original token count: 0
Output:


ps -p 3723465 -o pid,pcpu,pmem,etime,cmd


Output:
Chunk ID: 3da24e
Wall time: 0.0510 seconds
Process exited with code 0
Original token count: 53
Output:
    PID %CPU %MEM     ELAPSED CMD
3723465 39.5  1.4       03:28 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000


Output: write_stdin failed: stdin is closed for this session; rerun exec_command with tty=true to keep stdin open


I’m moving to a clean benchmark run now using the existing non-streaming path and fixed timing payload (default JSON output includes per-request timings + inflight + GPU snapshot). I’ll keep max tokens low (128) so we get rapid, repeatable runs for concurrency levels.


VEENA3_TIMING_LOGS=true NUMBA_DISABLE_JIT=1 python3 scripts/stress_test_optimized_local.py --levels 1,2,4,8 --text short --max-tokens 128 --output /tmp/bench_short_ns_lvl_1_2_4_8.json


Output:
Chunk ID: 7a386f
Wall time: 30.0020 seconds
Process running with session ID 73073
Original token count: 0
Output:


Output:
Chunk ID: 29681b
Wall time: 30.0009 seconds
Process running with session ID 73073
Original token count: 0
Output:


Output:
Chunk ID: 3e05d1
Wall time: 30.0015 seconds
Process running with session ID 73073
Original token count: 0
Output:


ps -p 3723465 -o pid,pcpu,pmem,etime,cmd && nvidia-smi --query-compute-apps=pid,name,used_gpu_memory --format=csv,noheader,nounits


Output:
Chunk ID: c5cb19
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 66
Output:
    PID %CPU %MEM     ELAPSED CMD
3723465 26.9  1.4       05:05 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000
3723573, VLLM::EngineCore, 8994
3723465, python3, 414


Output:
Chunk ID: b1dec4
Wall time: 0.0508 seconds
Process exited with code 1
Original token count: 8
Output:
    PID %CPU     ELAPSED CMD


Output:
Chunk ID: 605cf8
Wall time: 0.0511 seconds
Process exited with code 0
Original token count: 37
Output:
3725283 python3 scripts/stress_test_optimized_local.py --levels 1,2,4,8 --text short --max-tokens 128 --output /tmp/bench_short_ns_lvl_1_2_4_8.json


ps -p 3725283 -o pid,pcpu,pmem,etime,cmd


Output:
Chunk ID: 11433a
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 51
Output:
    PID %CPU %MEM     ELAPSED CMD
3725283  0.4  0.0       01:39 python3 scripts/stress_test_optimized_local.py --levels 1,2,4,8 --text short --max-tokens 128 --output /tmp/bench_short_ns_lvl_1_2_4_8.json


Output:
Chunk ID: c71608
Wall time: 30.0019 seconds
Process running with session ID 73073
Original token count: 0
Output:


ls -l /tmp/bench_short_ns_lvl_1_2_4_8.json || true && tail -n 2 /tmp/bench_short_ns_lvl_1_2_4_8.json 2>/dev/null || true


Output:
Chunk ID: 8a916e
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 21
Output:
ls: cannot access '/tmp/bench_short_ns_lvl_1_2_4_8.json': No such file or directory


The benchmark loop is taking longer than expected because each request currently waits a full 120s by default, which hides failures and delays results. I’m going to stop that long-running run and add a `--timeout-seconds` option so each request fails fast and the concurrency sweep returns actionable metrics quickly.


Output:
Chunk ID: b9ff43
Wall time: 0.0508 seconds
Process exited with code 0
Original token count: 0
Output:


Input:
*** Begin Patch
*** Update File: scripts/stress_test_optimized_local.py
@@
 async def make_request(
     client: httpx.AsyncClient,
     base_url: str,
     text: str,
@@
     chunking: bool,
     max_tokens: int,
     stream: bool = False,
-    timeout: float = 120.0,
+    timeout: float = 120.0,
 ) -> RequestResult:
@@
                 return await tts_runtime_nonstream(
@@
                     json=payload,
                     headers={"Content-Type": "application/json"},
                     timeout=timeout,
                 )
@@
 async def run_level(
     base_url: str,
     num_requests: int,
@@
     chunking: bool,
     stream: bool,
     max_tokens: int,
+    request_timeout: float,
     gpu_monitor: Optional[GPUMonitor] = None,
 ) -> LevelResult:
@@
             return await make_request(
                 client=client,
                 base_url=base_url,
                 text=text,
                 speaker=speaker,
                 chunking=chunking,
                 max_tokens=max_tokens,
                 stream=stream,
+                timeout=request_timeout,
             )
@@
 async def run_stress_test(args):
@@
     warmup = await run_level(
         base_url=base_url,
         num_requests=1,
         concurrency=1,
         text_category=args.text,
         chunking=args.chunking,
         stream=False,
         max_tokens=args.max_tokens,
+        request_timeout=args.timeout_seconds,
         gpu_monitor=gpu_monitor,
     )
@@
         level = await run_level(
             base_url=base_url,
             num_requests=num_requests,
             concurrency=conc,
             text_category=args.text,
             chunking=args.chunking,
             stream=args.stream,
             max_tokens=args.max_tokens,
+            request_timeout=args.timeout_seconds,
             gpu_monitor=gpu_monitor,
         )
@@
 def main():
@@
     parser.add_argument("--stream", action="store_true", help="Use streaming endpoint")
+    parser.add_argument(
+        "--timeout-seconds",
+        type=float,
+        default=120.0,
+        help="Per-request timeout (seconds) for benchmark traffic",
+    )
     parser.add_argument("--output", default="stress_test_optimized_local.json", help="Output JSON path")
*** End Patch


Output:
apply_patch verification failed: Failed to find expected lines in /home/ubuntu/veenaModal/scripts/stress_test_optimized_local.py:
                return await tts_runtime_nonstream(


Input:
*** Begin Patch
*** Update File: scripts/stress_test_optimized_local.py
@@
 async def run_level(
     base_url: str,
     num_requests: int,
     concurrency: int,
     text_category: str,
     chunking: bool,
     stream: bool,
     max_tokens: int,
+    request_timeout: float,
     gpu_monitor: Optional[GPUMonitor] = None,
 ) -> LevelResult:
@@
                 text=text,
                 speaker=speaker,
                 chunking=chunking,
                 max_tokens=max_tokens,
                 stream=stream,
+                timeout=request_timeout,
             )
@@
     warmup = await run_level(
         base_url=base_url,
         num_requests=1,
         concurrency=1,
         text_category=args.text,
         chunking=args.chunking,
         stream=False,
         max_tokens=args.max_tokens,
+        request_timeout=args.timeout_seconds,
         gpu_monitor=gpu_monitor,
     )
@@
         level = await run_level(
             base_url=base_url,
             num_requests=num_requests,
             concurrency=conc,
             text_category=args.text,
             chunking=args.chunking,
             stream=args.stream,
             max_tokens=args.max_tokens,
+            request_timeout=args.timeout_seconds,
             gpu_monitor=gpu_monitor,
         )
@@
     parser.add_argument("--stream", action="store_true", help="Use streaming endpoint")
+    parser.add_argument(
+        "--timeout-seconds",
+        type=float,
+        default=120.0,
+        help="Per-request timeout (seconds) for benchmark traffic",
+    )
     parser.add_argument("--output", default="stress_test_optimized_local.json", help="Output JSON path")
*** End Patch


Output: {"output":"Success. Updated the following files:\nM scripts/stress_test_optimized_local.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


VEENA3_TIMING_LOGS=true NUMBA_DISABLE_JIT=1 python3 scripts/stress_test_optimized_local.py --levels 1,2,4,8 --text short --max-tokens 128 --timeout-seconds 45 --output /tmp/bench_short_ns_lvl_1_2_4_8_t45.json


Output:
Chunk ID: c7fe6b
Wall time: 30.0015 seconds
Process running with session ID 50015
Original token count: 0
Output:


Output:
Chunk ID: 2f14c3
Wall time: 30.0008 seconds
Process running with session ID 50015
Original token count: 0
Output:


pgrep -af 'stress_test_optimized_local.py --levels 1,2,4,8 --text short --max-tokens 128 --timeout-seconds 45'


Output:
Chunk ID: d954ab
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 44
Output:
3727036 python3 scripts/stress_test_optimized_local.py --levels 1,2,4,8 --text short --max-tokens 128 --timeout-seconds 45 --output /tmp/bench_short_ns_lvl_1_2_4_8_t45.json


curl -sS --max-time 5 http://localhost:8000/v1/tts/health


Output:
Chunk ID: 3f551f
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 33
Output:
{"status":"healthy","model_loaded":true,"model_version":"LLM","uptime_seconds":377.86,"gpu_available":true,"app_version":"0.1.0"}


Summary: auto


I’m seeing the benchmark still not returning intermediate output, so I’m validating a single non-stream request directly with a stricter per-call timeout to confirm whether generation is currently hanging or just very slow at the first level.


curl -sS -m 35 -o /tmp/tts_one.wav -D - -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": false, "max_tokens": 128}' | sed -n '1,120p'


Output:
Chunk ID: efad9c
Wall time: 30.0014 seconds
Process running with session ID 64224
Original token count: 0
Output:


Output:
Chunk ID: d8d863
Wall time: 3.7879 seconds
Process exited with code 0
Original token count: 20
Output:
curl: (28) Operation timed out after 35000 milliseconds with 0 bytes received


nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv,noheader,nounits && nvidia-smi pmon -c 1


Output:
Chunk ID: 6e5299
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 83
Output:
0, 9425
# gpu         pid   type     sm    mem    enc    dec    jpg    ofa    command
# Idx           #    C/G      %      %      %      %      %      %    name
    0    3723465     C      -      -      -      -      -      -    python3
    0    3723573     C      0      0      -      -      -      -    VLLM::EngineCor


curl -sS -m 180 -o /tmp/tts_one.wav -D - -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": false, "max_tokens": 128}' | sed -n '1,40p'


Output:
Chunk ID: 3b60e6
Wall time: 30.0019 seconds
Process running with session ID 69254
Original token count: 0
Output:


Output:
Chunk ID: 37a3f7
Wall time: 30.0005 seconds
Process running with session ID 69254
Original token count: 0
Output:


Output:
Chunk ID: 6b9a0d
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 0
Output:


pgrep -af 'curl -sS -m 180 -o /tmp/tts_one.wav'


Output:
Chunk ID: 31830f
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 114
Output:
3728397 /bin/bash -lc curl -sS -m 180 -o /tmp/tts_one.wav -D - -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": false, "max_tokens": 128}' | sed -n '1,40p'
3728401 curl -sS -m 180 -o /tmp/tts_one.wav -D - -X POST http://localhost:8000/v1/tts/generate -H Content-Type: application/json -d {"text":"Hello", "speaker":"lipakshi", "chunking": false, "max_tokens": 128}


curl -sS -m 60 -N -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": false, "max_tokens": 128, "stream": true}' --output /tmp/tts_stream.wav


Output:
Chunk ID: bf4497
Wall time: 30.0019 seconds
Process running with session ID 92400
Original token count: 0
Output:


Output:
Chunk ID: 48a5f2
Wall time: 28.9340 seconds
Process exited with code 28
Original token count: 20
Output:
curl: (28) Operation timed out after 60001 milliseconds with 0 bytes received


The endpoint is now confirming a deeper issue: even a single request returns no bytes within 60s in both streaming and non-streaming modes. I’ll do a quick focused check of the local server launch options to capture live logs from requests and confirm where it is stalling.


sed -n '1,260p' veena3modal/local_server.py


Output:
Chunk ID: 0a96e3
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 2380
Output:
"""
Local entry point for Veena3 TTS - runs the same FastAPI app without Modal.

Replicates what Modal's @modal.enter + @modal.asgi_app does:
1. Sets up PYTHONPATH for vendored deps (sparktts, AP-BWE)
2. Downloads model weights from HuggingFace if not present
3. Initializes TTS runtime (vLLM engine, BiCodec decoder, pipelines)
4. Serves the FastAPI app via uvicorn

Usage:
    # Basic (auto-downloads model, uses defaults)
    python -m veena3modal.local_server

    # Custom model path (skip download)
    python -m veena3modal.local_server --model-path /path/to/spark_tts_4speaker

    # Custom GPU memory + port
    python -m veena3modal.local_server --gpu-memory 0.5 --port 8080

    # With super-resolution (48kHz output)
    python -m veena3modal.local_server --enable-sr --sr-path /path/to/ap_bwe/16kto48k

    # CPU-only (no GPU, for testing only - very slow)
    python -m veena3modal.local_server --device cpu

Environment Variables (override defaults):
    MODEL_PATH              - Path to Spark TTS model directory
    HF_TOKEN                - HuggingFace token for private models
    AP_BWE_CHECKPOINT_DIR   - Path to super-resolution checkpoints
    AUTH_BYPASS_MODE        - "true" to disable auth (default for local)
    GPU_MEMORY_UTILIZATION  - vLLM GPU memory fraction (default: 0.25)
"""

from __future__ import annotations

import argparse
import logging
import os
import sys
import time

# === Path Setup (MUST happen before any veena3modal imports) ===
# Vendored deps: sparktts and AP-BWE live in external/ relative to repo root
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
EXTERNAL_DIR = os.path.join(REPO_ROOT, "external")

# sparktts package path: external/sparktts (contains sparktts/ subpackage)
SPARKTTS_PATH = os.path.join(EXTERNAL_DIR, "sparktts")
# AP-BWE path: external/AP-BWE
AP_BWE_PATH = os.path.join(EXTERNAL_DIR, "AP-BWE")

for path in [SPARKTTS_PATH, AP_BWE_PATH, REPO_ROOT]:
    if path not in sys.path:
        sys.path.insert(0, path)

# Also set PYTHONPATH for subprocesses (e.g., ffmpeg wrappers)
existing_pythonpath = os.environ.get("PYTHONPATH", "")
new_paths = f"{SPARKTTS_PATH}:{AP_BWE_PATH}:{REPO_ROOT}"
if existing_pythonpath:
    os.environ["PYTHONPATH"] = f"{new_paths}:{existing_pythonpath}"
else:
    os.environ["PYTHONPATH"] = new_paths

logger = logging.getLogger("veena3.local")

# Default local model directory (relative to repo)
DEFAULT_LOCAL_MODEL_DIR = os.path.join(REPO_ROOT, "models", "spark_tts_4speaker")

# Model sources (tried in order):
# 1. Local path (--model-path or MODEL_PATH env)
# 2. HuggingFace download (private repo: BayAreaBoys/spark_tts_4speaker)
# 3. Base Spark TTS model (public: SparkAudio/Spark-TTS-0.5B)
PRIVATE_HF_REPO = "BayAreaBoys/spark_tts_4speaker"
PUBLIC_FALLBACK_REPO = "SparkAudio/Spark-TTS-0.5B"

def download_model(
    hf_repo: str = PRIVATE_HF_REPO,
    local_dir: str = DEFAULT_LOCAL_MODEL_DIR,
    hf_token: str | None = None,
) -> str:
    """
    Download Spark TTS model from HuggingFace if not already present.

    Strategy:
    1. Try private repo (BayAreaBoys/spark_tts_4speaker) with HF token
    2. Fall back to public base model (SparkAudio/Spark-TTS-0.5B)
    3. Provide instructions for Modal volume download if both fail

    Returns the resolved local path.
    """
    # Check for any valid marker file (config.json at root, config.yaml, or LLM/config.json)
    marker_files = ["config.json", "config.yaml", "LLM/config.json"]
    if any(os.path.exists(os.path.join(local_dir, m)) for m in marker_files):
        logger.info(f"Model already present at {local_dir}")
        return local_dir

    try:
        from huggingface_hub import snapshot_download
    except ImportError:
        logger.error(
            "huggingface_hub not installed. Install with: pip install huggingface-hub\n"
            "Or download model manually and pass --model-path"
        )
        sys.exit(1)

    os.makedirs(local_dir, exist_ok=True)

    # Try private repo first
    logger.info(f"Attempting download from {hf_repo}...")
    try:
        downloaded_path = snapshot_download(
            repo_id=hf_repo,
            local_dir=local_dir,
            token=hf_token,
            ignore_patterns=["*.bin", "training_args.bin", "optimizer.pt"],
        )
        logger.info(f"Model downloaded to {downloaded_path}")
        return downloaded_path
    except Exception as e:
        logger.warning(f"Private repo download failed: {e}")

    # Try public fallback
    if hf_repo != PUBLIC_FALLBACK_REPO:
        logger.info(f"Trying public base model: {PUBLIC_FALLBACK_REPO}...")
        try:
            downloaded_path = snapshot_download(
                repo_id=PUBLIC_FALLBACK_REPO,
                local_dir=local_dir,
                token=hf_token,
            )
            logger.info(f"Base model downloaded to {downloaded_path}")
            logger.warning(
                "Using public base SparkTTS model (no custom speakers). "
                "For full speaker support, download your fine-tuned model."
            )
            return downloaded_path
        except Exception as e2:
            logger.warning(f"Public model download also failed: {e2}")

    # All downloads failed - provide manual instructions
    logger.error(
        "\nModel download failed. Options:\n"
        "  1. Download from Modal volume:\n"
        "     modal volume get veena3-models spark_tts_4speaker models/spark_tts_4speaker\n\n"
        "  2. Download from HuggingFace (if you have access):\n"
        f"     huggingface-cli download {PRIVATE_HF_REPO} --local-dir {local_dir}\n\n"
        "  3. Use base SparkTTS model:\n"
        f"     huggingface-cli download {PUBLIC_FALLBACK_REPO} --local-dir {local_dir}\n\n"
        "  4. Specify a custom path:\n"
        "     python -m veena3modal.local_server --model-path /path/to/model"
    )
    sys.exit(1)

def resolve_model_paths(model_path: str) -> tuple[str, str]:
    """
    Resolve LLM and BiCodec paths from a Spark TTS model directory.

    Spark TTS model structure (from SparkAudio/Spark-TTS-0.5B):
      model_path/
        LLM/          <- Language model (vLLM loads this)
        BiCodec/      <- Audio tokenizer/decoder
        config.yaml   <- Top-level config

    Fine-tuned models may have LLM files at root (flat structure).
    This function auto-detects both layouts.

    Returns:
        (llm_path, bicodec_path) resolved paths
    """
    # Check for subdirectory layout (SparkAudio/Spark-TTS-0.5B style)
    llm_subdir = os.path.join(model_path, "LLM")
    bicodec_subdir = os.path.join(model_path, "BiCodec")

    if os.path.isdir(llm_subdir) and os.path.exists(os.path.join(llm_subdir, "config.json")):
        llm_path = llm_subdir
    else:
        # Flat layout: LLM files at root
        llm_path = model_path

    # BiCodec path is always the root (it contains BiCodec/ subdir)
    bicodec_path = model_path

    logger.info(f"Resolved model paths:")
    logger.info(f"  LLM (vLLM):    {llm_path}")
    logger.info(f"  BiCodec:       {bicodec_path}")

    return llm_path, bicodec_path

def initialize_local_runtime(
    model_path: str,
    device: str = "cuda",
    gpu_memory_utilization: float = 0.85,
    enable_sr: bool = False,
    sr_checkpoint_dir: str | None = None,
    hf_token: str | None = None,
    num_engines: int = 1,
):
    """
    Initialize TTS runtime for local serving.

    This replaces Modal's @modal.enter lifecycle hook.
    Loads: vLLM engine, BiCodec decoder, prompt builder, pipelines.
    """
    from veena3modal.services.tts_runtime import initialize_runtime

    # Resolve LLM vs BiCodec paths from model directory
    llm_path, bicodec_path = resolve_model_paths(model_path)

    start = time.time()
    logger.info("=" * 60)
    logger.info("Initializing Veena3 TTS Runtime (local)")
    logger.info(f"  Model:       {model_path}")
    logger.info(f"  LLM path:    {llm_path}")
    logger.info(f"  BiCodec:     {bicodec_path}")
    logger.info(f"  Device:      {device}")
    logger.info(f"  GPU Memory:  {gpu_memory_utilization:.0%} (total)")
    logger.info(f"  Engines:     {num_engines}")
    logger.info(f"  SR:          {'enabled' if enable_sr else 'disabled'}")
    logger.info("=" * 60)

    runtime = initialize_runtime(
        model_path=llm_path,
        bicodec_path=bicodec_path,
        sr_checkpoint_dir=sr_checkpoint_dir if enable_sr else None,
        device=device,
        hf_token=hf_token,
        gpu_memory_utilization=gpu_memory_utilization,
        enable_sr=enable_sr,
        num_engines=num_engines,
    )

    elapsed = time.time() - start
    logger.info(f"Runtime initialized in {elapsed:.1f}s")
    logger.info(f"  Model version: {runtime.model_version}")
    logger.info(f"  Streaming: {'available' if runtime.streaming_pipeline else 'unavailable'}")
    logger.info(f"  Super-resolution: {'loaded' if runtime.sr_service and runtime.sr_service.is_loaded else 'disabled'}")

    return runtime

def create_local_app():
    """
    Create the FastAPI app for local serving.

    Same app factory as Modal deployment, but called from local context.
    """
    from veena3modal.api.fastapi_app import create_app
    return create_app()

def parse_args():
    """Parse CLI arguments for local server configuration."""
    parser = argparse.ArgumentParser(
        description="Veena3 TTS Local Server - GPU-accelerated text-to-speech",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:


sed -n '260,620p' veena3modal/local_server.py


Output:
Chunk ID: bd4e56
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 1660
Output:
Examples:
  python -m veena3modal.local_server                          # Auto-download model, serve on :8000
  python -m veena3modal.local_server --port 8080              # Custom port
  python -m veena3modal.local_server --model-path ./my_model  # Use local model
  python -m veena3modal.local_server --gpu-memory 0.5         # Limit GPU memory
  python -m veena3modal.local_server --workers 2              # Multiple workers (careful: each loads model)
        """,
    )

    # Model configuration
    model_group = parser.add_argument_group("Model")
    model_group.add_argument(
        "--model-path",
        type=str,
        default=os.environ.get("MODEL_PATH", DEFAULT_LOCAL_MODEL_DIR),
        help=f"Path to Spark TTS model (default: {DEFAULT_LOCAL_MODEL_DIR})",
    )
    model_group.add_argument(
        "--hf-repo",
        type=str,
        default=PRIVATE_HF_REPO,
        help=f"HuggingFace repo to download from (default: {PRIVATE_HF_REPO})",
    )
    model_group.add_argument(
        "--hf-token",
        type=str,
        default=os.environ.get("HF_TOKEN"),
        help="HuggingFace token (default: from HF_TOKEN env var)",
    )
    model_group.add_argument(
        "--skip-download",
        action="store_true",
        help="Don't auto-download model (fail if not found)",
    )

    # GPU configuration
    gpu_group = parser.add_argument_group("GPU")
    gpu_group.add_argument(
        "--device",
        type=str,
        default="cuda",
        choices=["cuda", "cpu"],
        help="Device for inference (default: cuda)",
    )
    gpu_group.add_argument(
        "--gpu-memory",
        type=float,
        default=float(os.environ.get("GPU_MEMORY_UTILIZATION", "0.25")),
        help="vLLM GPU memory utilization fraction (default: 0.25)",
    )
    gpu_group.add_argument(
        "--num-engines",
        type=int,
        default=int(os.environ.get("NUM_ENGINES", "1")),
        help="Number of vLLM engine instances (default: 1, use 2-3 for multi-engine mode)",
    )

    # Super-resolution
    sr_group = parser.add_argument_group("Super-Resolution")
    sr_group.add_argument(
        "--enable-sr",
        action="store_true",
        default=False,
        help="Enable AP-BWE super-resolution (16kHz -> 48kHz)",
    )
    sr_group.add_argument(
        "--sr-path",
        type=str,
        default=os.environ.get("AP_BWE_CHECKPOINT_DIR"),
        help="Path to AP-BWE checkpoint directory",
    )

    # Server configuration
    server_group = parser.add_argument_group("Server")
    server_group.add_argument(
        "--host",
        type=str,
        default="0.0.0.0",
        help="Bind address (default: 0.0.0.0)",
    )
    server_group.add_argument(
        "--port",
        type=int,
        default=int(os.environ.get("PORT", "8000")),
        help="Port to serve on (default: 8000)",
    )
    server_group.add_argument(
        "--workers",
        type=int,
        default=1,
        help="Number of uvicorn workers (default: 1, each loads full model)",
    )
    server_group.add_argument(
        "--reload",
        action="store_true",
        default=False,
        help="Enable auto-reload for development (incompatible with --workers > 1)",
    )

    # Auth
    auth_group = parser.add_argument_group("Auth")
    auth_group.add_argument(
        "--auth",
        action="store_true",
        default=False,
        help="Enable API key authentication (default: disabled for local)",
    )

    # Logging
    log_group = parser.add_argument_group("Logging")
    log_group.add_argument(
        "--log-level",
        type=str,
        default="INFO",
        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
        help="Log level (default: INFO)",
    )

    return parser.parse_args()

def main():
    """
    Main entry point for local TTS server.

    Lifecycle:
    1. Parse args + configure logging
    2. Download model if needed
    3. Initialize TTS runtime (vLLM engine, decoders, pipelines)
    4. Create FastAPI app
    5. Run uvicorn
    """
    args = parse_args()

    # Configure logging
    logging.basicConfig(
        level=getattr(logging, args.log_level),
        format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )

    # Auth bypass for local development (unless --auth flag is set)
    if not args.auth:
        os.environ["AUTH_BYPASS_MODE"] = "true"
        logger.info("Auth bypass enabled (local mode). Use --auth to require API keys.")
    else:
        os.environ.setdefault("AUTH_BYPASS_MODE", "false")

    # Resolve model path
    # Spark TTS root has config.yaml; LLM subdir has config.json; fine-tuned may have config.json at root
    model_path = args.model_path
    model_exists = any(
        os.path.exists(os.path.join(model_path, marker))
        for marker in ("config.json", "config.yaml", "LLM/config.json")
    )
    if not model_exists:
        if args.skip_download:
            logger.error(f"Model not found at {model_path} and --skip-download is set")
            sys.exit(1)
        model_path = download_model(
            hf_repo=args.hf_repo,
            local_dir=model_path,
            hf_token=args.hf_token,
        )

    # Initialize TTS runtime (this is the expensive step - loads model onto GPU)
    initialize_local_runtime(
        model_path=model_path,
        device=args.device,
        gpu_memory_utilization=args.gpu_memory,
        enable_sr=args.enable_sr,
        sr_checkpoint_dir=args.sr_path,
        hf_token=args.hf_token,
        num_engines=args.num_engines,
    )

    # Run uvicorn
    # NOTE: We use the factory string so uvicorn can find the app.
    # With --reload, uvicorn reimports, but the runtime singleton persists in memory.
    import uvicorn

    logger.info("")
    logger.info("=" * 60)
    logger.info(f"Veena3 TTS Local Server starting")
    logger.info(f"  URL: http://{args.host}:{args.port}")
    logger.info(f"  Health: http://{args.host}:{args.port}/v1/tts/health")
    logger.info(f"  Generate: POST http://{args.host}:{args.port}/v1/tts/generate")
    logger.info(f"  Metrics: http://{args.host}:{args.port}/v1/tts/metrics")
    logger.info(f"  Docs: http://{args.host}:{args.port}/docs")
    logger.info("=" * 60)

    uvicorn.run(
        "veena3modal.local_server:create_local_app",
        factory=True,
        host=args.host,
        port=args.port,
        workers=args.workers,
        reload=args.reload,
        log_level=args.log_level.lower(),
        # Performance: keep-alive for connection reuse under load
        timeout_keep_alive=30,
    )

if __name__ == "__main__":
    main()


kill 3723465 && pkill -f 'VLLM::EngineCore' || true


Output:
Chunk ID: aa5828
Wall time: 0.0519 seconds
Process exited with code -1
Original token count: 0
Output:


pgrep -af 'python3 -m veena3modal.local_server' || true && nvidia-smi --query-compute-apps=pid,name,used_gpu_memory --format=csv,noheader,nounits


Output:
Chunk ID: d45106
Wall time: 0.0504 seconds
Process exited with code 0
Original token count: 86
Output:
3723465 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000
3730091 /bin/bash -lc pgrep -af 'python3 -m veena3modal.local_server' || true && nvidia-smi --query-compute-apps=pid,name,used_gpu_memory --format=csv,noheader,nounits
3723465, python3, 414


kill -9 3723465 && sleep 1; pgrep -af 'python3 -m veena3modal.local_server' || true


Output:
Chunk ID: cc4e1c
Wall time: 0.9168 seconds
Process exited with code 0
Original token count: 27
Output:
3730119 /bin/bash -lc kill -9 3723465 && sleep 1; pgrep -af 'python3 -m veena3modal.local_server' || true


Output:
Chunk ID: d8d8ee
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 0
Output:


Output:
Chunk ID: f7289c
Wall time: 0.0509 seconds
Process exited with code 0
Original token count: 0
Output:


VEENA3_TIMING_LOGS=true NUMBA_DISABLE_JIT=1 VEENA3_REQUEST_TIMEOUT_SECONDS=20 BICODEC_DEVICE=cpu python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000


Output:
Chunk ID: 4eb870
Wall time: 30.0006 seconds
Process running with session ID 79754
Original token count: 3065
Output:
2026-02-17 01:08:52 | INFO     | veena3.local | Auth bypass enabled (local mode). Use --auth to require API keys.
2026-02-17 01:08:52 | INFO     | veena3.local | Resolved model paths:
2026-02-17 01:08:52 | INFO     | veena3.local |   LLM (vLLM):    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:08:52 | INFO     | veena3.local |   BiCodec:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:08:52 | INFO     | veena3.local | ============================================================
2026-02-17 01:08:52 | INFO     | veena3.local | Initializing Veena3 TTS Runtime (local)
2026-02-17 01:08:52 | INFO     | veena3.local |   Model:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:08:52 | INFO     | veena3.local |   LLM path:    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:08:52 | INFO     | veena3.local |   BiCodec:     /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:08:52 | INFO     | veena3.local |   Device:      cuda
2026-02-17 01:08:52 | INFO     | veena3.local |   GPU Memory:  10% (total)
2026-02-17 01:08:52 | INFO     | veena3.local |   Engines:     1
2026-02-17 01:08:52 | INFO     | veena3.local |   SR:          disabled
2026-02-17 01:08:52 | INFO     | veena3.local | ============================================================
2026-02-17 01:08:52 | INFO     | veena3modal.services.tts_runtime | Initializing TTS runtime...
2026-02-17 01:08:52 | INFO     | veena3modal.services.tts_runtime |   Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:08:52 | INFO     | veena3modal.services.tts_runtime |   BiCodec path: /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:08:52 | INFO     | veena3modal.services.tts_runtime |   BiCodec device: cpu
2026-02-17 01:08:52 | INFO     | veena3modal.services.tts_runtime |   SR enabled: False, path: None
2026-02-17 01:08:57 | INFO     | veena3modal.services.tts_runtime | Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.1
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-17 01:08:58 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 01:08:58 [model.py:1561] Using max model len 4096
INFO 02-17 01:08:58 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 01:08:58 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3730344) INFO 02-17 01:08:59 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:02 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:41307 backend=nccl
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:02 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:02 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:03 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=3730344) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=3730344) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=3730344)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=3730344)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.37s/it]
(EngineCore_DP0 pid=3730344)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:01<00:00,  1.37s/it]
(EngineCore_DP0 pid=3730344)
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:04 [default_loader.py:291] Loading weights took 1.43 seconds
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:05 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.985584 seconds
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:10 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:10 [backends.py:872] Dynamo bytecode transform time: 4.17 s
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:13 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.701 s
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:13 [monitor.py:34] torch.compile takes 4.87 s in total
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:14 [gpu_worker.py:356] Available KV cache memory: 6.17 GiB
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:14 [kv_cache_utils.py:1307] GPU KV cache size: 538,928 tokens
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:14 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 131.57x
(EngineCore_DP0 pid=3730344)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 32.85it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 34.70it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 36.19it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 36.97it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 37.26it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 37.34it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 37.27it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 35.63it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:00<00:00, 35.21it/s]
(EngineCore_DP0 pid=3730344)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 29.72it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 33.93it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 33.86it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 34.73it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.85it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 34.86it/s]
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:16 [gpu_model_runner.py:5063] Graph capturing finished in 3 secs, took 0.28 GiB
(EngineCore_DP0 pid=3730344) INFO 02-17 01:09:16 [core.py:272] init engine (profile, create kv cache, warmup model) took 11.12 seconds
2026-02-17 01:09:18 | INFO     | veena3modal.services.tts_runtime | Initializing prompt builder...
2026-02-17 01:09:18 | INFO     | veena3modal.services.tts_runtime | Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)


Output:
Chunk ID: 35c2ed
Wall time: 30.0010 seconds
Process running with session ID 79754
Original token count: 1013
Output:
2026-02-17 01:09:25 | INFO     | veena3modal.services.tts_runtime | Initializing TTS pipeline...
2026-02-17 01:09:25 | INFO     | veena3modal.services.tts_runtime | Initializing streaming pipeline...
2026-02-17 01:09:25 | INFO     | veena3modal.services.tts_runtime | ✅ TTS runtime initialized in 33024ms
2026-02-17 01:09:25 | INFO     | veena3modal.services.tts_runtime |    Model version: LLM
2026-02-17 01:09:25 | INFO     | veena3modal.services.tts_runtime | Pre-computing global tokens for all speakers...
2026-02-17 01:09:26 | INFO     | veena3modal.services.tts_runtime |   Cached globals for lipakshi
2026-02-17 01:09:26 | INFO     | veena3modal.services.tts_runtime |   Cached globals for vardan
2026-02-17 01:09:26 | INFO     | veena3modal.services.tts_runtime |   Cached globals for reet
2026-02-17 01:09:27 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nandini
2026-02-17 01:09:27 | INFO     | veena3modal.services.tts_runtime |   Cached globals for krishna
2026-02-17 01:09:28 | INFO     | veena3modal.services.tts_runtime |   Cached globals for anika
2026-02-17 01:09:28 | INFO     | veena3modal.services.tts_runtime |   Cached globals for adarsh
2026-02-17 01:09:28 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nilay
2026-02-17 01:09:29 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Aarvi
2026-02-17 01:09:29 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Asha
2026-02-17 01:09:30 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Bittu
2026-02-17 01:09:30 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Mira
2026-02-17 01:09:30 | INFO     | veena3modal.services.tts_runtime | Speaker globals cached: 12/12 speakers in 4799ms
2026-02-17 01:09:30 | INFO     | veena3.local | Runtime initialized in 37.8s
2026-02-17 01:09:30 | INFO     | veena3.local |   Model version: LLM
2026-02-17 01:09:30 | INFO     | veena3.local |   Streaming: available
2026-02-17 01:09:30 | INFO     | veena3.local |   Super-resolution: disabled
2026-02-17 01:09:30 | INFO     | veena3.local |
2026-02-17 01:09:30 | INFO     | veena3.local | ============================================================
2026-02-17 01:09:30 | INFO     | veena3.local | Veena3 TTS Local Server starting
2026-02-17 01:09:30 | INFO     | veena3.local |   URL: http://0.0.0.0:8000
2026-02-17 01:09:30 | INFO     | veena3.local |   Health: http://0.0.0.0:8000/v1/tts/health
2026-02-17 01:09:30 | INFO     | veena3.local |   Generate: POST http://0.0.0.0:8000/v1/tts/generate
2026-02-17 01:09:30 | INFO     | veena3.local |   Metrics: http://0.0.0.0:8000/v1/tts/metrics
2026-02-17 01:09:30 | INFO     | veena3.local |   Docs: http://0.0.0.0:8000/docs
2026-02-17 01:09:30 | INFO     | veena3.local | ============================================================
✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cpu
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
🚀 SparkTTSPipeline initialized
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)
{"timestamp": "2026-02-17T01:09:30.465581+00:00", "level": "INFO", "logger": "veena3modal.api.websocket_handler", "message": "WebSocket route /v1/tts/ws registered"}
2026-02-17 01:09:30 | INFO     | veena3modal.api.websocket_handler | WebSocket route /v1/tts/ws registered
2026-02-17 01:09:30 | INFO     | veena3modal.api.fastapi_app | WebSocket support enabled at /v1/tts/ws
{"timestamp": "2026-02-17T01:09:30.465713+00:00", "level": "INFO", "logger": "veena3modal.api.fastapi_app", "message": "WebSocket support enabled at /v1/tts/ws"}
INFO:     Started server process [3730211]
INFO:     Waiting for application startup.
INFO:     Application startup complete.
INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)


curl -sS -m 60 -o /tmp/tts_one.wav -D - -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": false, "max_tokens": 128}'


Output:
Chunk ID: 29c098
Wall time: 19.9418 seconds
Process exited with code 0
Original token count: 50
Output:
HTTP/1.1 500 Internal Server Error
date: Tue, 17 Feb 2026 01:09:55 GMT
server: uvicorn
x-request-id: 4fe1721c-a59a-416a-93c8-a6e418c0f638
content-length: 144
content-type: application/json


Output:
Chunk ID: 89da8a
Wall time: 30.0013 seconds
Process running with session ID 79754
Original token count: 1783
Output:
{"timestamp": "2026-02-17T01:09:55.914553+00:00", "level": "INFO", "logger": "veena3modal.tts", "message": "request_received", "request_id": "4fe1721c-a59a-416a-93c8-a6e418c0f638", "event": "request_received", "text_length": 5, "speaker": "lipakshi", "stream": false, "format": "wav"}
2026-02-17 01:09:55 | INFO     | veena3modal.tts | request_received
2026-02-17 01:09:55 | INFO     | veena3modal.services.tts_runtime | tts_runtime_nonstream_start
2026-02-17 01:09:55 | INFO     | veena3modal.core.pipeline | tts_llm_generate_start
2026-02-17 01:10:15 | ERROR    | veena3modal.services.tts_runtime | Speech generation failed: Non-stream generation timed out after 20.00s
2026-02-17 01:10:15 | ERROR    | veena3modal.services.tts_runtime | tts_runtime_nonstream_error
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 401, in generate_speech_indic
    return await self.generate_speech(
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 177, in generate_speech
    request_output = await iterator.__anext__()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 572, in generate
    out = q.get_nowait() or await q.get()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/output_processor.py", line 76, in get
    await self.ready.wait()
  File "/usr/lib/python3.10/asyncio/locks.py", line 214, in wait
    await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.10/asyncio/tasks.py", line 456, in wait_for
    return fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 485, in generate_speech
    audio_bytes, pipeline_timings = await asyncio.wait_for(
  File "/usr/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
    raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 490, in generate_speech
    raise TimeoutError(
TimeoutError: Non-stream generation timed out after 20.00s
2026-02-17 01:10:15 | INFO     | veena3modal.services.tts_runtime | tts_runtime_nonstream_complete
2026-02-17 01:10:15 | INFO     | veena3modal.services.tts_runtime | tts_runtime_nonstream_inflight
{"timestamp": "2026-02-17T01:10:15.916826+00:00", "level": "ERROR", "logger": "veena3modal.api.fastapi_app", "message": "TTS generation error: Non-stream generation timed out after 20.00s", "request_id": "4fe1721c-a59a-416a-93c8-a6e418c0f638", "exception": "Traceback (most recent call last):\n  File \"/home/ubuntu/veenaModal/veena3modal/core/pipeline.py\", line 401, in generate_speech_indic\n    return await self.generate_speech(\n  File \"/home/ubuntu/veenaModal/veena3modal/core/pipeline.py\", line 177, in generate_speech\n    request_output = await iterator.__anext__()\n  File \"/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py\", line 572, in generate\n    out = q.get_nowait() or await q.get()\n  File \"/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/output_processor.py\", line 76, in get\n    await self.ready.wait()\n  File \"/usr/lib/python3.10/asyncio/locks.py\", line 214, in wait\n    await fut\nasyncio.exceptions.CancelledError\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n    return fut.result()\nasyncio.exceptions.CancelledError\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\", line 485, in generate_speech\n    audio_bytes, pipeline_timings = await asyncio.wait_for(\n  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n    raise exceptions.TimeoutError() from exc\nasyncio.exceptions.TimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py\", line 460, in tts_generate\n    audio_bytes, metrics = await tts_runtime.generate_speech(\n  File \"/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py\", line 490, in generate_speech\n    raise TimeoutError(\nTimeoutError: Non-stream generation timed out after 20.00s\n"}
2026-02-17 01:10:15 | ERROR    | veena3modal.api.fastapi_app | TTS generation error: Non-stream generation timed out after 20.00s
Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 401, in generate_speech_indic
    return await self.generate_speech(
  File "/home/ubuntu/veenaModal/veena3modal/core/pipeline.py", line 177, in generate_speech
    request_output = await iterator.__anext__()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/async_llm.py", line 572, in generate
    out = q.get_nowait() or await q.get()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/engine/output_processor.py", line 76, in get
    await self.ready.wait()
  File "/usr/lib/python3.10/asyncio/locks.py", line 214, in wait
    await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.10/asyncio/tasks.py", line 456, in wait_for
    return fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 485, in generate_speech
    audio_bytes, pipeline_timings = await asyncio.wait_for(
  File "/usr/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
    raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/ubuntu/veenaModal/veena3modal/api/fastapi_app.py", line 460, in tts_generate
    audio_bytes, metrics = await tts_runtime.generate_speech(
  File "/home/ubuntu/veenaModal/veena3modal/services/tts_runtime.py", line 490, in generate_speech
    raise TimeoutError(
TimeoutError: Non-stream generation timed out after 20.00s
{"timestamp": "2026-02-17T01:10:15.917359+00:00", "level": "ERROR", "logger": "veena3modal.tts", "message": "request_failed", "request_id": "4fe1721c-a59a-416a-93c8-a6e418c0f638", "event": "request_failed", "status_code": 500, "error_code": "INTERNAL_ERROR", "error_message": "Non-stream generation timed out after 20.00s"}
2026-02-17 01:10:15 | ERROR    | veena3modal.tts | request_failed
INFO:     127.0.0.1:45952 - "POST /v1/tts/generate HTTP/1.1" 500 Internal Server Error


Output: write_stdin failed: stdin is closed for this session; rerun exec_command with tty=true to keep stdin open


I identified that request handling is stalling at `await iterator.__anext__` under non-stream mode. I’m going to compare behavior with timing logs disabled to isolate whether this is a generation timeout/path issue versus instrumentation impact, then proceed with a benchmark run that returns deterministic data.


pgrep -af 'python3 -m veena3modal.local_server'


Output:
Chunk ID: 2960d1
Wall time: 0.0514 seconds
Process exited with code 0
Original token count: 39
Output:
3730211 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000


Summary: auto


kill -9 3730211 && sleep 1; pgrep -af 'python3 -m veena3modal.local_server' || true


Output:
Chunk ID: 6e3e44
Wall time: 0.9184 seconds
Process exited with code 0
Original token count: 27
Output:
3730856 /bin/bash -lc kill -9 3730211 && sleep 1; pgrep -af 'python3 -m veena3modal.local_server' || true


pkill -f VLLM::EngineCore || true; pgrep -af 'VLLM::EngineCore' || true


Output:
Chunk ID: 64a676
Wall time: 0.0514 seconds
Process exited with code -1
Original token count: 0
Output:


BICODEC_DEVICE=cpu NUMBA_DISABLE_JIT=1 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000


Output:
Chunk ID: 4ea67a
Wall time: 30.0016 seconds
Process running with session ID 98130
Original token count: 3193
Output:
Total output lines: 100

2026-02-17 01:10:56 | INFO     | veena3.local | Auth bypass enabled (local mode). Use --auth to require API keys.
2026-02-17 01:10:56 | INFO     | veena3.local | Resolved model paths:
2026-02-17 01:10:56 | INFO     | veena3.local |   LLM (vLLM):    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:10:56 | INFO     | veena3.local |   BiCodec:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:10:56 | INFO     | veena3.local | ============================================================
2026-02-17 01:10:56 | INFO     | veena3.local | Initializing Veena3 TTS Runtime (local)
2026-02-17 01:10:56 | INFO     | veena3.local |   Model:       /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:10:56 | INFO     | veena3.local |   LLM path:    /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:10:56 | INFO     | veena3.local |   BiCodec:     /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:10:56 | INFO     | veena3.local |   Device:      cuda
2026-02-17 01:10:56 | INFO     | veena3.local |   GPU Memory:  10% (total)
2026-02-17 01:10:56 | INFO     | veena3.local |   Engines:     1
2026-02-17 01:10:56 | INFO     | veena3.local |   SR:          disabled
2026-02-17 01:10:56 | INFO     | veena3.local | ============================================================
2026-02-17 01:10:56 | INFO     | veena3modal.services.tts_runtime | Initializing TTS runtime...
2026-02-17 01:10:56 | INFO     | veena3modal.services.tts_runtime |   Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
2026-02-17 01:10:56 | INFO     | veena3modal.services.tts_runtime |   BiCodec path: /home/ubuntu/veenaModal/models/spark_tts_4speaker
2026-02-17 01:10:56 | INFO     | veena3modal.services.tts_runtime |   BiCodec device: cpu
2026-02-17 01:10:56 | INFO     | veena3modal.services.tts_runtime |   SR enabled: False, path: None
2026-02-17 01:11:01 | INFO     | veena3modal.services.tts_runtime | Loading SparkTTS model with vLLM...
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
🚀 Initializing Spark TTS Model
📁 Model path: /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM
🗣  Speakers: 12 predefined (lipakshi, vardan, reet, Nandini, krishna, anika, adarsh, Nilay, Aarvi, Asha, Bittu, Mira)
🎭 Emotions: 10 tags in [bracket] format

📝 Loading tokenizer from model...
✅ Tokenizer loaded: 165168 tokens

🔍 Validating 12 speakers for Spark TTS...
  lipakshi → <|speaker_0|>: 7 tokens
  vardan → <|speaker_1|>: 7 tokens
  reet → <|speaker_2|>: 7 tokens
  Nandini → <|speaker_3|>: 7 tokens
  krishna → <|speaker_4|>: 7 tokens
  anika → <|speaker_5|>: 7 tokens
  adarsh → <|speaker_6|>: 7 tokens
  Nilay → <|speaker_7|>: 7 tokens
  Aarvi → <|speaker_8|>: 7 tokens
  Asha → <|speaker_9|>: 7 tokens
  Bittu → <|speaker_10|>: 8 tokens
  Mira → <|speaker_11|>: 8 tokens
✅ All 12 speakers validated with mapping

🔍 Validating 10 emotion tags for Spark TTS...
✅ All 10 emotion tags validated

📋 Sample emotion tags:
  [angry]: 1 tokens
  [curious]: 1 tokens
  [excited]: 1 tokens
  [giggle]: 1 tokens
  [laughs harder]: 1 tokens
  ...

🔧 Initializing vLLM engine with optimizations...
  - dtype: bfloat16
  - max_model_len: 4096
  - gpu_memory_utilization: 0.1
  - enable_prefix_caching: True
  - enforce_eager (disable CUDA graphs): False
INFO 02-17 01:11:02 [model.py:541] Resolved architecture: Qwen2ForCausalLM
INFO 02-17 01:11:02 [model.py:1561] Using max model len 4096
INFO 02-17 01:11:02 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=4096.
INFO 02-17 01:11:02 [vllm.py:624] Asynchronous scheduling is enabled.
The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:03 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', speculative_config=None, tokenizer='/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'co…193 tokens truncated…s': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:05 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://216.81.248.184:37599 backend=nccl
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:05 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:06 [gpu_model_runner.py:4033] Starting to load model /home/ubuntu/veenaModal/models/spark_tts_4speaker/LLM...
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:07 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
(EngineCore_DP0 pid=3731006) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.cudart module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.runtime module instead.
(EngineCore_DP0 pid=3731006) <frozen importlib._bootstrap_external>:1184: FutureWarning: The cuda.nvrtc module is deprecated and will be removed in a future release, please switch to use the cuda.bindings.nvrtc module instead.
(EngineCore_DP0 pid=3731006)
Loading safetensors checkpoint shards:   0% Completed | 0/1 [00:00<?, ?it/s]
(EngineCore_DP0 pid=3731006)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.26it/s]
(EngineCore_DP0 pid=3731006)
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00,  1.26it/s]
(EngineCore_DP0 pid=3731006)
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:08 [default_loader.py:291] Loading weights took 0.86 seconds
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:08 [gpu_model_runner.py:4130] Model loading took 0.95 GiB memory and 1.396015 seconds
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:13 [backends.py:812] Using cache directory: /home/ubuntu/.cache/vllm/torch_compile_cache/f3ff13511c/rank_0_0/backbone for vLLM's torch.compile
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:13 [backends.py:872] Dynamo bytecode transform time: 4.21 s
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:16 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 4096) from the cache, took 0.642 s
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:16 [monitor.py:34] torch.compile takes 4.86 s in total
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:17 [gpu_worker.py:356] Available KV cache memory: 6.17 GiB
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:17 [kv_cache_utils.py:1307] GPU KV cache size: 538,928 tokens
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:17 [kv_cache_utils.py:1312] Maximum concurrency for 4,096 tokens per request: 131.57x
(EngineCore_DP0 pid=3731006)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  11%|█▏        | 4/35 [00:00<00:00, 32.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  23%|██▎       | 8/35 [00:00<00:00, 35.98it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  34%|███▍      | 12/35 [00:00<00:00, 36.09it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  46%|████▌     | 16/35 [00:00<00:00, 36.42it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  57%|█████▋    | 20/35 [00:00<00:00, 36.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  69%|██████▊   | 24/35 [00:00<00:00, 37.03it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 28/35 [00:00<00:00, 36.44it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  91%|█████████▏| 32/35 [00:00<00:00, 35.11it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 35/35 [00:00<00:00, 35.08it/s]
(EngineCore_DP0 pid=3731006)
Capturing CUDA graphs (decode, FULL):   0%|          | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL):  16%|█▌        | 3/19 [00:00<00:00, 29.52it/s]
Capturing CUDA graphs (decode, FULL):  37%|███▋      | 7/19 [00:00<00:00, 34.15it/s]
Capturing CUDA graphs (decode, FULL):  58%|█████▊    | 11/19 [00:00<00:00, 35.29it/s]
Capturing CUDA graphs (decode, FULL):  79%|███████▉  | 15/19 [00:00<00:00, 35.88it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 36.42it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 19/19 [00:00<00:00, 35.55it/s]
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:19 [gpu_model_runner.py:5063] Graph capturing finished in 2 secs, took 0.28 GiB
(EngineCore_DP0 pid=3731006) INFO 02-17 01:11:19 [core.py:272] init engine (profile, create kv cache, warmup model) took 10.89 seconds
2026-02-17 01:11:21 | INFO     | veena3modal.services.tts_runtime | Initializing prompt builder...
2026-02-17 01:11:21 | INFO     | veena3modal.services.tts_runtime | Initializing BiCodec decoder...
/home/ubuntu/.local/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)
2026-02-17 01:11:26 | INFO     | veena3modal.services.tts_runtime | Initializing TTS pipeline...
2026-02-17 01:11:26 | INFO     | veena3modal.services.tts_runtime | Initializing streaming pipeline...
2026-02-17 01:11:26 | INFO     | veena3modal.services.tts_runtime | ✅ TTS runtime initialized in 29764ms
2026-02-17 01:11:26 | INFO     | veena3modal.services.tts_runtime |    Model version: LLM
2026-02-17 01:11:26 | INFO     | veena3modal.services.tts_runtime | Pre-computing global tokens for all speakers...


Output:
Chunk ID: c1c63b
Wall time: 30.0016 seconds
Process running with session ID 98130
Original token count: 885
Output:
2026-02-17 01:11:26 | INFO     | veena3modal.services.tts_runtime |   Cached globals for lipakshi
2026-02-17 01:11:27 | INFO     | veena3modal.services.tts_runtime |   Cached globals for vardan
2026-02-17 01:11:27 | INFO     | veena3modal.services.tts_runtime |   Cached globals for reet
2026-02-17 01:11:27 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nandini
2026-02-17 01:11:28 | INFO     | veena3modal.services.tts_runtime |   Cached globals for krishna
2026-02-17 01:11:28 | INFO     | veena3modal.services.tts_runtime |   Cached globals for anika
2026-02-17 01:11:29 | INFO     | veena3modal.services.tts_runtime |   Cached globals for adarsh
2026-02-17 01:11:29 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Nilay
2026-02-17 01:11:29 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Aarvi
2026-02-17 01:11:30 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Asha
2026-02-17 01:11:30 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Bittu
2026-02-17 01:11:31 | INFO     | veena3modal.services.tts_runtime |   Cached globals for Mira
2026-02-17 01:11:31 | INFO     | veena3modal.services.tts_runtime | Speaker globals cached: 12/12 speakers in 4879ms
2026-02-17 01:11:31 | INFO     | veena3.local | Runtime initialized in 34.6s
2026-02-17 01:11:31 | INFO     | veena3.local |   Model version: LLM
2026-02-17 01:11:31 | INFO     | veena3.local |   Streaming: available
2026-02-17 01:11:31 | INFO     | veena3.local |   Super-resolution: disabled
2026-02-17 01:11:31 | INFO     | veena3.local |
2026-02-17 01:11:31 | INFO     | veena3.local | ============================================================
2026-02-17 01:11:31 | INFO     | veena3.local | Veena3 TTS Local Server starting
2026-02-17 01:11:31 | INFO     | veena3.local |   URL: http://0.0.0.0:8000
2026-02-17 01:11:31 | INFO     | veena3.local |   Health: http://0.0.0.0:8000/v1/tts/health
2026-02-17 01:11:31 | INFO     | veena3.local |   Generate: POST http://0.0.0.0:8000/v1/tts/generate
2026-02-17 01:11:31 | INFO     | veena3.local |   Metrics: http://0.0.0.0:8000/v1/tts/metrics
2026-02-17 01:11:31 | INFO     | veena3.local |   Docs: http://0.0.0.0:8000/docs
2026-02-17 01:11:31 | INFO     | veena3.local | ============================================================
✅ vLLM engine initialized successfully!

🎉 Spark TTS Model ready for inference!

🎵 Loading BiCodec audio tokenizer from /home/ubuntu/veenaModal/models/spark_tts_4speaker...
   Device: cpu
Missing tensor: mel_transformer.spectrogram.window
Missing tensor: mel_transformer.mel_scale.fb
✅ BiCodec decoder initialized (sample rate: 16000Hz)
🚀 SparkTTSPipeline initialized
🌊 Veena3SlidingWindowPipeline initialized (sliding window: 28 tokens)
{"timestamp": "2026-02-17T01:11:31.187354+00:00", "level": "INFO", "logger": "veena3modal.api.websocket_handler", "message": "WebSocket route /v1/tts/ws registered"}
2026-02-17 01:11:31 | INFO     | veena3modal.api.websocket_handler | WebSocket route /v1/tts/ws registered
{"timestamp": "2026-02-17T01:11:31.187491+00:00", "level": "INFO", "logger": "veena3modal.api.fastapi_app", "message": "WebSocket support enabled at /v1/tts/ws"}
2026-02-17 01:11:31 | INFO     | veena3modal.api.fastapi_app | WebSocket support enabled at /v1/tts/ws
INFO:     Started server process [3730907]
INFO:     Waiting for application startup.
INFO:     Application startup complete.
INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)


curl -sS -m 120 -o /tmp/tts_one.wav -D - -X POST http://localhost:8000/v1/tts/generate -H 'Content-Type: application/json' -d '{"text":"Hello", "speaker":"lipakshi", "chunking": false, "max_tokens": 128}'


Output:
Chunk ID: b6c6e1
Wall time: 30.0014 seconds
Process running with session ID 49990
Original token count: 0
Output:


Output:
Chunk ID: af5e3c
Wall time: 30.0009 seconds
Process running with session ID 49990
Original token count: 0
Output:


Output:
Chunk ID: 61771f
Wall time: 30.0005 seconds
Process running with session ID 49990
Original token count: 0
Output:


Output:
Chunk ID: 036f4e
Wall time: 5.0005 seconds
Process running with session ID 98130
Original token count: 113
Output:
{"timestamp": "2026-02-17T01:11:59.013941+00:00", "level": "INFO", "logger": "veena3modal.tts", "message": "request_received", "request_id": "230a46c2-1cdd-4479-b223-28d769bb5197", "event": "request_received", "text_length": 5, "speaker": "lipakshi", "stream": false, "format": "wav"}
2026-02-17 01:11:59 | INFO     | veena3modal.tts | request_received
2026-02-17 01:11:59 | INFO     | veena3modal.services.tts_runtime | tts_runtime_nonstream_start


Output:
Chunk ID: c0fce6
Wall time: 15.0012 seconds
Process running with session ID 98130
Original token count: 0
Output:


Output:
Chunk ID: 2f024b
Wall time: 0.0515 seconds
Process exited with code 0
Original token count: 83
Output:
0, 9423
# gpu         pid   type     sm    mem    enc    dec    jpg    ofa    command
# Idx           #    C/G      %      %      %      %      %      %    name
    0    3730907     C      -      -      -      -      -      -    python3
    0    3731006     C      0      0      -      -      -      -    VLLM::EngineCor


ps -p 3730907 -o pid,pcpu,pmem,etime,cmd; ps -p 3731006 -o pid,pcpu,pmem,etime,cmd


Output:
Chunk ID: b14a90
Wall time: 0.0513 seconds
Process exited with code 0
Original token count: 73
Output:
    PID %CPU %MEM     ELAPSED CMD
3730907 37.1  1.5       03:01 python3 -m veena3modal.local_server --model-path /home/ubuntu/veenaModal/models/spark_tts_4speaker --gpu-memory 0.1 --log-level INFO --port 8000
    PID %CPU %MEM     ELAPSED CMD
3731006 20.0  1.6       02:54 VLLM::EngineCore


curl -sS http://localhost:8000/v1/tts/metrics | head -n 80


Output:
Chunk ID: c9fbef
Wall time: 0.0512 seconds
Process exited with code 0
Original token count: 333
Output:
# HELP veena3_tts_requests_total Total TTS requests received
# TYPE veena3_tts_requests_total counter
veena3_tts_requests_total{format="AudioFormat_WAV",speaker="lipakshi",stream="false"} 1.0
# HELP veena3_tts_requests_created Total TTS requests received
# TYPE veena3_tts_requests_created gauge
veena3_tts_requests_created{format="AudioFormat_WAV",speaker="lipakshi",stream="false"} 1.7712907190143936e+09
# HELP veena3_tts_requests_completed_total Total TTS requests completed successfully
# TYPE veena3_tts_requests_completed_total counter
# HELP veena3_tts_requests_failed_total Total TTS requests failed
# TYPE veena3_tts_requests_failed_total counter
# HELP veena3_tts_ttfb_seconds Time to first byte in seconds
# TYPE veena3_tts_ttfb_seconds histogram
# HELP veena3_tts_rtf Real-time factor (generation_time / audio_duration)
# TYPE veena3_tts_rtf histogram
# HELP veena3_tts_audio_duration_seconds Generated audio duration in seconds
# TYPE veena3_tts_audio_duration_seconds histogram
# HELP veena3_tts_chunks_sent Number of audio chunks sent in streaming mode
# TYPE veena3_tts_chunks_sent histogram
# HELP veena3_tts_model_load_seconds Time taken to load the model
# TYPE veena3_tts_model_load_seconds gauge
# HELP veena3_tts_model_loaded Whether the model is loaded (1) or not (0)
# TYPE veena3_tts_model_loaded gauge


sed -n '1,260p' veena3modal/core/pipeline.py