Output:
Exit code: 0
Wall time: 0 seconds
Output:
            )

        inputs = self._processor(wav, sampling_rate=sr, return_tensors="pt")
        input_features = inputs.get("input_features")
        if input_features is None:
            raise RuntimeError("Processor did not return input_features")
        input_features = input_features.to(self.device)
        model_dtype = getattr(self._model, "dtype", None)
        if model_dtype is not None:
            input_features = input_features.to(dtype=model_dtype)

        forced_decoder_ids = None
        whisper_lang = to_whisper_lang(language)
        if whisper_lang:
            get_ids = getattr(self._processor, "get_decoder_prompt_ids", None)
            if callable(get_ids):
                try:
                    forced_decoder_ids = get_ids(language=whisper_lang, task="transcribe")
                except (ValueError, KeyError):
                    forced_decoder_ids = None

        with self._torch.no_grad():
            gen_kwargs = {
                "max_new_tokens": max_new_tokens,
                "num_beams": num_beams,
            }
            if forced_decoder_ids is not None:
                gen_kwargs["forced_decoder_ids"] = forced_decoder_ids
            try:
                generated_ids = self._model.generate(input_features, **gen_kwargs)
            except ValueError as e:
                # Some fine-tuned checkpoints / transformer versions reject `forced_decoder_ids`.
                msg = str(e)
                if "forced_decoder_ids" in msg and "not used by the model" in msg:
                    # Whisper's newer generate() prefers forced ids via generation_config.
                    gen_kwargs.pop("forced_decoder_ids", None)
                    old_forced = None
                    restore_forced = False
                    if forced_decoder_ids is not None:
                        gen_cfg = getattr(self._model, "generation_config", None)
                        if gen_cfg is not None:
                            old_forced = getattr(gen_cfg, "forced_decoder_ids", None)
                            gen_cfg.forced_decoder_ids = forced_decoder_ids
                            restore_forced = True
                    try:
                        generated_ids = self._model.generate(input_features, **gen_kwargs)
                    finally:
                        if restore_forced:
                            gen_cfg = getattr(self._model, "generation_config", None)
                            if gen_cfg is not None:
                                gen_cfg.forced_decoder_ids = old_forced
                else:
                    raise

        text = self._processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        return ASRResult(
            model=self.name,
            text=text.strip(),
            language=whisper_lang or language,
            language_probability=None,
            segments=[],
            meta={
                "device": self.device,
                "dtype": self.dtype,
                "max_new_tokens": max_new_tokens,
                "num_beams": num_beams,
            },
        )

class TransformersWhisperASRRouter:
    """
    Lazily load and cache Whisper-family HF checkpoints by language.
    Useful for per-language fine-tuned models without running 10+ models per segment.
    """

    def __init__(
        self,
        *,
        model_map: dict[str, str],
        default_model: str | None = None,
        device: str | None = None,
        dtype: str | None = None,
        max_loaded_models: int = 1,
    ) -> None:
        self.model_map = {str(k).strip().lower(): str(v).strip() for k, v in (model_map or {}).items() if str(v).strip()}
        self.default_model = default_model.strip() if isinstance(default_model, str) and default_model.strip() else None
        self.device = device
        self.dtype = dtype
        self.max_loaded_models = max(1, int(max_loaded_models))
        self._cache: OrderedDict[str, TransformersWhisperASR] = OrderedDict()

    def _get_model_id(self, language: str | None) -> str | None:
        if language:
            key = language.strip().lower()
            if key in self.model_map:
                return self.model_map[key]
        return self.default_model

    def transcribe(
        self,
        audio_path: Path,
        *,
        language: str | None = None,
        max_new_tokens: int = 256,
        num_beams: int = 5,
    ) -> ASRResult | None:
        model_id = self._get_model_id(language)
        if not model_id:
            return None

        backend = self._cache.get(model_id)
        if backend is None:
            backend = TransformersWhisperASR(model_id=model_id, device=self.device, dtype=self.dtype)
            self._cache[model_id] = backend
        else:
            # Refresh LRU ordering.
            self._cache.move_to_end(model_id, last=True)

        while len(self._cache) > self.max_loaded_models:
            _, evicted = self._cache.popitem(last=False)
            try:
                evicted.close()
            except Exception:
                pass

        return backend.transcribe(
            audio_path=audio_path,
            language=language,
            max_new_tokens=max_new_tokens,
            num_beams=num_beams,
        )

def _to_iso3(lang: str | None) -> str | None:
    """Convert language code to ISO-639-3 format."""
    if not lang:
        return None
    code = lang.strip().lower()
    if not code:
        return None
    code = code.replace("-", "_")
    if "_" in code:
        code = code.split("_", 1)[0]
    if len(code) == 3:
        return code
    if len(code) == 2:
        return _WHISPER_TO_ISO3.get(code, None)
    return code

class MMSASR:
    """
    MMS ASR backend using CTC decoding (facebook/mms-1b-all).
    Provides architectural diversity from Whisper's encoder-decoder approach.
    """

    def __init__(
        self,
        model_id: str = "facebook/mms-1b-all",
        device: str | None = None,
    ) -> None:
        try:
            import torch
            from transformers import AutoModelForCTC, AutoProcessor
        except Exception as e:  # pragma: no cover
            raise RuntimeError(
                "MMSASR requires torch + transformers. Install with: pip install -e '.[torch]'"
            ) from e

        if device is None:
            device = "cuda" if torch.cuda.is_available() else "cpu"

        self.model_id = model_id
        self.device = device
        self._torch = torch
        self._processor = AutoProcessor.from_pretrained(model_id)
        self._model = AutoModelForCTC.from_pretrained(model_id).to(device)
        self._model.eval()

        # MMS models require set_target_lang for language-specific vocab.
        self._target_lang: str | None = None

    @property
    def name(self) -> str:
        return f"mms-ctc:{self.model_id.split('/')[-1]}"

    def close(self) -> None:
        """Best-effort release of GPU memory."""
        model = getattr(self, "_model", None)
        if model is not None:
            try:
                del self._model
            except Exception:
                pass
        proc = getattr(self, "_processor", None)
        if proc is not None:
            try:
                del self._processor
            except Exception:
                pass

        try:
            gc.collect()
        except Exception:
            pass

        try:
            if self.device == "cuda" and hasattr(self, "_torch"):
                self._torch.cuda.empty_cache()
        except Exception:
            pass

    def _maybe_set_target_lang(self, lang: str | None) -> None:
        """Set target language for MMS multilingual tokenizer."""
        tokenizer = getattr(self._processor, "tokenizer", None)
        set_lang = getattr(tokenizer, "set_target_lang", None) if tokenizer is not None else None
        if not callable(set_lang):
            return

        iso3 = _to_iso3(lang)
        if not iso3:
            return
        if self._target_lang == iso3:
            return
        try:
            set_lang(iso3)
            self._target_lang = iso3
        except Exception:
            # Language not supported by model, continue without setting.
            pass

    def transcribe(
        self,
        audio_path: Path,
        language: str | None = None,
    ) -> ASRResult:
        from maya_transcribe.audio import decode_pcm16

        wav, sr = decode_pcm16(audio_path, sample_rate=16000, mono=True)
        if wav.size == 0:
            return ASRResult(
                model=self.name,
                text="",
                language=language,
                language_probability=None,
                segments=[],
                meta={"device": self.device},
            )

        # Set target language for multilingual vocab.
        self._maybe_set_target_lang(language)

        inputs = self._processor(wav, sampling_rate=sr, return_tensors="pt")
        input_values = inputs.get("input_values")
        if input_values is None:
            input_values = inputs.get("input_features")
        if input_values is None:
            raise RuntimeError("MMS processor did not return input_values or input_features")
        input_values = input_values.to(self.device)

        with self._torch.no_grad():
            logits = self._model(input_values).logits
            predicted_ids = self._torch.argmax(logits, dim=-1)

        text = self._processor.batch_decode(predicted_ids)[0]

        return ASRResult(
            model=self.name,
            text=text.strip(),
            language=self._target_lang or language,
            language_probability=None,
            segments=[],
            meta={"device": self.device, "target_lang": self._target_lang},
        )

class MMSASRRouter:
    """
    Router for MMS ASR with language-specific model loading.
    MMS-1B-all supports 1000+ languages with a single model but benefits from
    language-specific tokenizer configuration.
    """

    def __init__(
        self,
        *,
        model_id: str = "facebook/mms-1b-all",
        device: str | None = None,
    ) -> None:
        self.model_id = model_id
        self.device = device
        self._backend: MMSASR | None = None

    def transcribe(
        self,
        audio_path: Path,
        *,
        language: str | None = None,
    ) -> ASRResult | None:
        if self._backend is None:


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


sed -n '520,900p' maya_transcribe/asr.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
        if self._backend is None:
            self._backend = MMSASR(model_id=self.model_id, device=self.device)
        return self._backend.transcribe(audio_path=audio_path, language=language)

class IndicConformerASR:
    """
    IndicConformer ASR backend using NVIDIA NeMo.
    Provides Conformer architecture diversity from Whisper's encoder-decoder.
    Uses ai4bharat/indicconformer_stt_hi_hybrid_rnnt_large or similar checkpoints.
    """

    def __init__(
        self,
        model_name: str = "ai4bharat/indicconformer_stt_hi_hybrid_rnnt_large",
        device: str | None = None,
    ) -> None:
        try:
            import torch
            import nemo.collections.asr as nemo_asr
        except Exception as e:  # pragma: no cover
            raise RuntimeError(
                "IndicConformerASR requires nemo_toolkit. Install with: pip install nemo_toolkit[asr]"
            ) from e

        if device is None:
            device = "cuda" if torch.cuda.is_available() else "cpu"

        self.model_name = model_name
        self.device = device
        self._torch = torch

        # Load NeMo ASR model from HuggingFace or local path.
        self._model = nemo_asr.models.ASRModel.from_pretrained(model_name=model_name)
        if device == "cuda":
            self._model = self._model.cuda()
        self._model.eval()

    @property
    def name(self) -> str:
        return f"indicconformer:{self.model_name.split('/')[-1]}"

    def close(self) -> None:
        """Best-effort release of GPU memory."""
        model = getattr(self, "_model", None)
        if model is not None:
            try:
                del self._model
            except Exception:
                pass
        try:
            gc.collect()
        except Exception:
            pass
        try:
            if self.device == "cuda" and hasattr(self, "_torch"):
                self._torch.cuda.empty_cache()
        except Exception:
            pass

    def transcribe(
        self,
        audio_path: Path,
        language: str | None = None,
    ) -> ASRResult:
        # NeMo models transcribe directly from file path.
        try:
            texts = self._model.transcribe([str(audio_path)])
            text = texts[0] if texts else ""
        except Exception:
            text = ""

        return ASRResult(
            model=self.name,
            text=text.strip(),
            language=language,
            language_probability=None,
            segments=[],
            meta={"device": self.device},
        )

class IndicConformerASRRouter:
    """
    Router for IndicConformer that loads language-specific models on demand.
    Maps language codes to ai4bharat IndicConformer checkpoints.
    """

    # Default model map for IndicConformer checkpoints.
    DEFAULT_MODEL_MAP: dict[str, str] = {
        "hin": "ai4bharat/indicconformer_stt_hi_hybrid_rnnt_large",
        "hi": "ai4bharat/indicconformer_stt_hi_hybrid_rnnt_large",
        "ben": "ai4bharat/indicconformer_stt_bn_hybrid_rnnt_large",
        "bn": "ai4bharat/indicconformer_stt_bn_hybrid_rnnt_large",
        "tam": "ai4bharat/indicconformer_stt_ta_hybrid_rnnt_large",
        "ta": "ai4bharat/indicconformer_stt_ta_hybrid_rnnt_large",
        "tel": "ai4bharat/indicconformer_stt_te_hybrid_rnnt_large",
        "te": "ai4bharat/indicconformer_stt_te_hybrid_rnnt_large",
        "kan": "ai4bharat/indicconformer_stt_kn_hybrid_rnnt_large",
        "kn": "ai4bharat/indicconformer_stt_kn_hybrid_rnnt_large",
        "mal": "ai4bharat/indicconformer_stt_ml_hybrid_rnnt_large",
        "ml": "ai4bharat/indicconformer_stt_ml_hybrid_rnnt_large",
        "mar": "ai4bharat/indicconformer_stt_mr_hybrid_rnnt_large",
        "mr": "ai4bharat/indicconformer_stt_mr_hybrid_rnnt_large",
        "guj": "ai4bharat/indicconformer_stt_gu_hybrid_rnnt_large",
        "gu": "ai4bharat/indicconformer_stt_gu_hybrid_rnnt_large",
        "pan": "ai4bharat/indicconformer_stt_pa_hybrid_rnnt_large",
        "pa": "ai4bharat/indicconformer_stt_pa_hybrid_rnnt_large",
        "ory": "ai4bharat/indicconformer_stt_or_hybrid_rnnt_large",
        "or": "ai4bharat/indicconformer_stt_or_hybrid_rnnt_large",
        "asm": "ai4bharat/indicconformer_stt_as_hybrid_rnnt_large",
        "as": "ai4bharat/indicconformer_stt_as_hybrid_rnnt_large",
    }

    def __init__(
        self,
        *,
        model_map: dict[str, str] | None = None,
        default_model: str | None = None,
        device: str | None = None,
        max_loaded_models: int = 1,
    ) -> None:
        self.model_map = {str(k).strip().lower(): str(v).strip() for k, v in (model_map or self.DEFAULT_MODEL_MAP).items() if str(v).strip()}
        self.default_model = default_model.strip() if isinstance(default_model, str) and default_model.strip() else None
        self.device = device
        self.max_loaded_models = max(1, int(max_loaded_models))
        self._cache: OrderedDict[str, IndicConformerASR] = OrderedDict()

    def _get_model_name(self, language: str | None) -> str | None:
        if language:
            key = language.strip().lower()
            if key in self.model_map:
                return self.model_map[key]
        return self.default_model

    def transcribe(
        self,
        audio_path: Path,
        *,
        language: str | None = None,
    ) -> ASRResult | None:
        model_name = self._get_model_name(language)
        if not model_name:
            return None

        backend = self._cache.get(model_name)
        if backend is None:
            backend = IndicConformerASR(model_name=model_name, device=self.device)
            self._cache[model_name] = backend
        else:
            self._cache.move_to_end(model_name, last=True)

        while len(self._cache) > self.max_loaded_models:
            _, evicted = self._cache.popitem(last=False)
            try:
                evicted.close()
            except Exception:
                pass

        return backend.transcribe(audio_path=audio_path, language=language)


Summary: auto


**Reviewing ASR backends and model options**

I confirmed the current ASR backends don’t include vLLM and rely on transformers. The user prefers vLLM or other engines for models like Gemma3n or Voxtral, which are likely large multimodal models supporting audio. I’m clarifying if vLLM can run these audio transcription models and planning to verify the existence and details of these models, possibly via Hugging Face.


**Checking dependencies for Hugging Face Hub**

I'm inspecting pyproject.toml to see if huggingface_hub or related packages are installed, which would allow downloading models from Hugging Face as the user requested.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
[build-system]
requires = ["setuptools>=69", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "maya-transcribe"
version = "0.1.0"
description = "Quality-first offline transcription pipeline (Indic + English)"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
  "boto3>=1.34.0",
  "jiwer>=3.0.4",
  "numpy>=1.24.0",
  "python-dotenv>=1.0.1",
  "rich>=13.7.1",
  "typer>=0.12.3",
]

[project.optional-dependencies]
supabase = ["supabase>=2.6.0"]
db = ["psycopg2-binary>=2.9.9"]
gemini = ["google-genai>=1.54.0"]
bench = [
  "datasets>=2.19.0",
  "huggingface_hub>=0.23.0",
]
asr = [
  "faster-whisper>=1.0.3",
  "soundfile>=0.12.1",
]
torch = [
  "torch>=2.2.0",
  "torchaudio>=2.2.0",
  "transformers>=4.41.0",
]
# IndicConformer requires NVIDIA NeMo toolkit (per master.md plan for architecture diversity).
nemo = [
  "nemo_toolkit[asr]>=1.23.0",
]
full = [
  "datasets>=2.19.0",
  "huggingface_hub>=0.23.0",
  "supabase>=2.6.0",
  "psycopg2-binary>=2.9.9",
  "google-genai>=1.54.0",
  "faster-whisper>=1.0.3",
  "soundfile>=0.12.1",
  "torch>=2.2.0",
  "torchaudio>=2.2.0",
  "transformers>=4.41.0",
]

[project.scripts]
maya-transcribe = "maya_transcribe.cli:app"

[tool.setuptools.packages.find]
where = ["."]
include = ["maya_transcribe*"]


**Checking pipeline config for vLLM support**

I'm looking into the pipeline configuration in maya_transcribe/config.py to see if it references or supports vLLM, which might require adding a dependency for testing or usage.


sed -n '1,260p' maya_transcribe/config.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Final

from dotenv import load_dotenv

@dataclass(frozen=True)
class Settings:
    r2_endpoint_url: str
    r2_access_key_id: str
    r2_secret_access_key: str
    r2_bucket: str = "1-cleaned-data"
    r2_prefix: str | None = None

    @staticmethod
    def load(env_path: str | Path = ".env") -> "Settings":
        load_dotenv(dotenv_path=env_path, override=False)

        import os

        def required(name: str) -> str:
            value = os.getenv(name)
            if not value:
                raise RuntimeError(f"Missing required env var: {name}")
            return value

        endpoint_url: Final[str] = required("R2_ENDPOINT_URL")
        access_key_id: Final[str] = required("R2_ACCESS_KEY_ID")
        secret_access_key: Final[str] = required("R2_SECRET_ACCESS_KEY")
        bucket: Final[str] = os.getenv("R2_BUCKET") or "1-cleaned-data"
        prefix = os.getenv("R2_PREFIX") or None

        return Settings(
            r2_endpoint_url=endpoint_url,
            r2_access_key_id=access_key_id,
            r2_secret_access_key=secret_access_key,
            r2_bucket=bucket,
            r2_prefix=prefix,
        )


**Inspecting pipeline.py**


sed -n '1,260p' maya_transcribe/pipeline.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

import json
import time
from dataclasses import asdict
from pathlib import Path
from typing import Any

from rich.progress import track

from maya_transcribe.asr import (
    ASRResult,
    FasterWhisperASR,
    IndicConformerASRRouter,
    MMSASR,
    TransformersWhisperASR,
    TransformersWhisperASRRouter,
)
from maya_transcribe.audio import compute_qc, ffprobe, normalize_audio
from maya_transcribe.ids import segment_identity
from maya_transcribe.language import LANG_CODE_TO_NAME
from maya_transcribe.lid import LIDResult, MMSLID
from maya_transcribe.rover import rover_combine
from maya_transcribe.textnorm import normalize_text
from maya_transcribe.verify import verify_segment

def _iter_audio_files(segments_dir: Path) -> list[Path]:
    audio_files: list[Path] = []
    for ext in ("*.wav", "*.mp3", "*.m4a", "*.flac", "*.ogg", "*.opus"):
        audio_files.extend(sorted(segments_dir.rglob(ext)))
    return sorted(set(audio_files))

def _asr_summary(result: ASRResult) -> dict[str, Any]:
    avg_logprobs = [s.avg_logprob for s in result.segments if s.avg_logprob is not None]
    no_speech_probs = [s.no_speech_prob for s in result.segments if s.no_speech_prob is not None]
    compression_ratios = [s.compression_ratio for s in result.segments if s.compression_ratio is not None]

    def mean(xs: list[float]) -> float | None:
        return (sum(xs) / len(xs)) if xs else None

    return {
        "language": result.language,
        "language_probability": result.language_probability,
        "avg_logprob_mean": mean([float(x) for x in avg_logprobs]),
        "no_speech_prob_mean": mean([float(x) for x in no_speech_probs]),
        "compression_ratio_mean": mean([float(x) for x in compression_ratios]),
    }

def _infer_lang_from_relpath(relpath: str) -> str | None:
    parts = [p.strip().lower() for p in (relpath or "").split("/") if p.strip()]
    for p in parts[:-1]:
        if p in LANG_CODE_TO_NAME:
            return p
    return None

def run_pipeline(
    *,
    segments_dir: Path,
    out_dir: Path,
    lid_model_id: str = "facebook/mms-lid-256",
    whisper_model: str = "large-v3",
    extra_transformers_whisper_models: list[str] | None = None,
    transformers_whisper_model_map: dict[str, str] | None = None,
    transformers_whisper_default_model: str | None = None,
    # MMS ASR for CTC-based diversity (per master.md plan).
    mms_asr_model: str | None = None,
    # IndicConformer for Conformer architecture diversity (per master.md plan).
    indicconformer_model_map: dict[str, str] | None = None,
    indicconformer_default_model: str | None = None,
    gemini_model: str | None = None,
    gemini_prompt_path: Path | None = None,
    gemini_max_output_tokens: int = 512,
    gemini_hard_only: bool = True,
    ctc_model: str | None = None,
    ctc_model_map: dict[str, str] | None = None,
    require_lid_backend: bool = True,
    trim_silence: bool = True,
    silence_threshold_db: float = -40.0,
    min_silence_sec: float = 0.15,
    limit: int | None = None,
    thresholds: dict[str, Any] | None = None,
) -> Path:
    """
    End-to-end run: preprocess -> LID -> ASR ensemble -> normalize -> ROVER -> verify.
    Writes `out_dir/results.jsonl` and returns that path.
    """
    thresholds = thresholds or {}
    if (ctc_model or ctc_model_map) and "require_ctc" not in thresholds:
        thresholds["require_ctc"] = True
    if (ctc_model or ctc_model_map) and "min_ctc_margin" not in thresholds:
        thresholds["min_ctc_margin"] = 0.05
    out_dir.mkdir(parents=True, exist_ok=True)

    audio_files = _iter_audio_files(segments_dir)
    if not audio_files:
        raise RuntimeError(f"No audio files found under: {segments_dir}")
    if limit is not None:
        audio_files = audio_files[: max(0, int(limit))]

    # Instantiate models once.
    lid_backend: MMSLID | None
    try:
        lid_backend = MMSLID(model_id=lid_model_id)
    except Exception as e:
        if require_lid_backend:
            raise RuntimeError(
                "Failed to initialize MMS-LID. Install torch/transformers deps (pip install -e '.[torch]') "
                "or run with require_lid_backend=False."
            ) from e
        lid_backend = None

    whisper_backend = FasterWhisperASR(model_size_or_path=whisper_model)

    extra_transformers_whisper_models = extra_transformers_whisper_models or []
    transformers_backends: list[TransformersWhisperASR] = []
    for model_id in extra_transformers_whisper_models:
        transformers_backends.append(TransformersWhisperASR(model_id=model_id))

    transformers_router = None
    if transformers_whisper_model_map:
        transformers_router = TransformersWhisperASRRouter(
            model_map=transformers_whisper_model_map, default_model=transformers_whisper_default_model
        )

    # MMS ASR backend for CTC-based diversity (per master.md: "Optional diversity: MMS ASR").
    mms_backend: MMSASR | None = None
    if mms_asr_model:
        try:
            mms_backend = MMSASR(model_id=mms_asr_model)
        except Exception:
            pass  # MMS optional; continue if unavailable.

    # IndicConformer router for architecture diversity (per master.md: "IndicConformer-600M multilingual").
    indicconformer_router: IndicConformerASRRouter | None = None
    if indicconformer_model_map or indicconformer_default_model:
        try:
            indicconformer_router = IndicConformerASRRouter(
                model_map=indicconformer_model_map,
                default_model=indicconformer_default_model,
            )
        except Exception:
            pass  # IndicConformer optional; continue if NeMo unavailable.

    ctc_scorer = None
    rank_by_ctc = None
    if ctc_model or ctc_model_map:
        from maya_transcribe.ctc_fit import CTCFitScorer, rank_candidates_by_ctc

        ctc_scorer = CTCFitScorer(model_map=ctc_model_map, default_model=ctc_model)
        rank_by_ctc = rank_candidates_by_ctc

    gemini_backend = None
    if gemini_model:
        try:
            from maya_transcribe.gemini_asr import GeminiASR, GeminiConfig, load_prompt_template

            prompt = load_prompt_template(gemini_prompt_path or Path("Prompt.md"))
            gemini_backend = GeminiASR(
                config=GeminiConfig(model=gemini_model, max_output_tokens=gemini_max_output_tokens),
                prompt_template=prompt,
            )
        except Exception as e:
            raise RuntimeError("Failed to initialize Gemini backend (check GEMINI_KEY and google-genai).") from e

    default_whisper_run_sampling = not bool(transformers_backends or transformers_router or gemini_backend is not None)

    results_path = out_dir / "results.jsonl"
    summary_path = out_dir / "summary.json"

    counts = {"accept": 0, "human": 0, "reject": 0, "error": 0}
    timings_total: list[float] = []

    with results_path.open("w", encoding="utf-8") as f:
        for src in track(audio_files, description="Transcribing"):
            ident = segment_identity(src, root=segments_dir)
            segment_id = ident.segment_id
            t0 = time.perf_counter()

            try:
                # 1) Preprocess
                t_pre0 = time.perf_counter()
                src_info = ffprobe(src)
                wav_path = out_dir / "audio" / f"{segment_id}.wav"
                normalize_audio(
                    src,
                    wav_path,
                    trim_silence=trim_silence,
                    silence_threshold_db=silence_threshold_db,
                    min_silence_sec=min_silence_sec,
                )
                info = ffprobe(wav_path)
                qc = compute_qc(wav_path)
                t_pre1 = time.perf_counter()

                # 2) LID
                t_lid0 = time.perf_counter()
                lid: LIDResult | None = None
                if lid_backend is not None:
                    lid = lid_backend.predict(wav_path, top_k=5)
                t_lid1 = time.perf_counter()

                # Route language hint (only if confident).
                route_lang = None
                route_lang_source = None
                prefer_path = bool(thresholds.get("route_lang_prefer_path", False))
                if prefer_path and bool(thresholds.get("route_lang_from_path", False)):
                    inferred = _infer_lang_from_relpath(ident.relpath)
                    if inferred:
                        route_lang = inferred
                        route_lang_source = "path"

                if route_lang is None and lid and lid.predictions:
                    top1 = lid.predictions[0]
                    top2p = lid.predictions[1].prob if len(lid.predictions) > 1 else 0.0
                    if float(top1.prob) >= float(thresholds.get("lid_min_prob", 0.90)) and (
                        float(top1.prob) - float(top2p)
                    ) >= float(thresholds.get("lid_min_margin", 0.20)):
                        route_lang = top1.code
                        route_lang_source = "lid"

                if route_lang is None and bool(thresholds.get("route_lang_from_path", False)):
                    inferred = _infer_lang_from_relpath(ident.relpath)
                    if inferred:
                        route_lang = inferred
                        route_lang_source = "path"

                # 3) ASR ensemble
                t_asr0 = time.perf_counter()
                asr_results: dict[str, ASRResult] = {}
                asr_errors: dict[str, str] = {}
                primary_asr_key = None
                det = None
                router_key: str | None = None

                # Whisper: deterministic + a small sampling diversity pass.
                try:
                    det = whisper_backend.transcribe(
                        wav_path, language=route_lang, beam_size=5, best_of=5, temperature=0.0
                    )
                    primary_asr_key = f"{det.model}|det"
                    asr_results[primary_asr_key] = det
                except Exception as e:
                    asr_errors[f"{whisper_backend.name}|det"] = f"{type(e).__name__}:{e}"

                run_sampling = bool(thresholds.get("whisper_run_sampling", default_whisper_run_sampling))
                if run_sampling:
                    try:
                        samp = whisper_backend.transcribe(
                            wav_path, language=route_lang, beam_size=5, best_of=10, temperature=0.2
                        )
                        asr_results[f"{samp.model}|samp_t0.2"] = samp
                    except Exception as e:
                        asr_errors[f"{whisper_backend.name}|samp_t0.2"] = f"{type(e).__name__}:{e}"

                for tf_backend in transformers_backends:
                    try:


sed -n '260,620p' maya_transcribe/pipeline.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
                    try:
                        res = tf_backend.transcribe(wav_path, language=route_lang, num_beams=5, max_new_tokens=256)
                        asr_results[res.model] = res
                    except Exception as e:
                        asr_errors[tf_backend.name] = f"{type(e).__name__}:{e}"

                if transformers_router is not None:
                    try:
                        routed = transformers_router.transcribe(
                            wav_path,
                            language=route_lang,
                            num_beams=5,
                            max_new_tokens=256,
                        )
                        if routed is not None:
                            asr_results[routed.model] = routed
                            router_key = routed.model
                    except Exception as e:
                        asr_errors[f"transformers_router:{route_lang or 'none'}"] = f"{type(e).__name__}:{e}"

                # MMS ASR for CTC-based diversity (per master.md plan).
                if mms_backend is not None:
                    try:
                        mms_res = mms_backend.transcribe(wav_path, language=route_lang)
                        asr_results[mms_res.model] = mms_res
                    except Exception as e:
                        asr_errors[f"mms-ctc:{route_lang or 'none'}"] = f"{type(e).__name__}:{e}"

                # IndicConformer for Conformer architecture diversity (per master.md plan).
                if indicconformer_router is not None:
                    try:
                        conformer_res = indicconformer_router.transcribe(wav_path, language=route_lang)
                        if conformer_res is not None:
                            asr_results[conformer_res.model] = conformer_res
                    except Exception as e:
                        asr_errors[f"indicconformer:{route_lang or 'none'}"] = f"{type(e).__name__}:{e}"

                if primary_asr_key is None and asr_results:
                    primary_asr_key = sorted(asr_results.keys())[0]
                if not asr_results:
                    raise RuntimeError(f"no_asr_hypotheses:{asr_errors or 'unknown'}")

                # Gemini: optional extra hypothesis (prefer running only on "hard" segments).
                if gemini_backend is not None:
                    run_gemini = True
                    if gemini_hard_only:
                        base_for_heur = det or (next(iter(asr_results.values())) if asr_results else None)
                        det_sum = _asr_summary(base_for_heur) if base_for_heur is not None else {}
                        avg_lp = det_sum.get("avg_logprob_mean")
                        comp = det_sum.get("compression_ratio_mean")
                        no_speech = det_sum.get("no_speech_prob_mean")
                        run_gemini = bool(
                            route_lang is None
                            or (no_speech is not None and float(no_speech) > 0.5)
                            or (avg_lp is not None and float(avg_lp) < float(thresholds.get("min_avg_logprob_mean", -1.0)))
                            or (
                                comp is not None
                                and float(comp) > float(thresholds.get("max_compression_ratio_mean", 2.4))
                            )
                        )

                    if run_gemini:
                        try:
                            g = gemini_backend.transcribe(wav_path, language=route_lang)
                            asr_results[g.model] = g
                        except Exception:
                            pass

                t_asr1 = time.perf_counter()

                ensemble_models = sorted({v.model for v in asr_results.values()})
                ensemble_backends = sorted({v.model.split(":", 1)[0] for v in asr_results.values()})
                ensemble_warnings: list[str] = []
                if len(ensemble_models) < 2:
                    ensemble_warnings.append("low_model_diversity")

                # 4) Normalize hypotheses
                t_norm0 = time.perf_counter()
                norm_hyps: dict[str, str] = {}
                for k, res in asr_results.items():
                    lang_hint = res.language or route_lang
                    norm_hyps[k] = normalize_text(res.text, lang=lang_hint)
                t_norm1 = time.perf_counter()

                # 5) ROVER
                t_rover0 = time.perf_counter()
                rover_hyps = dict(norm_hyps)
                rover_filtered_out: list[str] = []
                if bool(thresholds.get("filter_hypotheses_by_script", False)) and route_lang:
                    from maya_transcribe.language import dominant_script, expected_script

                    expected = expected_script(route_lang)
                    if expected:
                        kept: dict[str, str] = {}
                        for k, res in asr_results.items():
                            script = dominant_script(res.text)
                            if script is None or script == expected:
                                kept[k] = norm_hyps.get(k, "")
                            else:
                                rover_filtered_out.append(k)

                        kept = {k: v for k, v in kept.items() if v.strip()}
                        if kept:
                            rover_hyps = kept
                        else:
                            rover_filtered_out = []

                include_sampling_opt = thresholds.get("rover_include_sampling")
                if include_sampling_opt is None:
                    include_sampling = len(ensemble_backends) < 2
                else:
                    include_sampling = bool(include_sampling_opt)

                if not include_sampling:
                    dropped = [k for k in rover_hyps.keys() if k.endswith("|samp_t0.2")]
                    for k in dropped:
                        rover_hyps.pop(k, None)
                    rover_filtered_out.extend(dropped)

                prefer_router_opt = thresholds.get("rover_prefer_router")
                prefer_router = bool(prefer_router_opt) if prefer_router_opt is not None else (router_key is not None)
                if prefer_router and router_key and router_key in rover_hyps:
                    rover_hyps = {
                        router_key: rover_hyps[router_key],
                        **{k: v for k, v in rover_hyps.items() if k != router_key},
                    }

                if primary_asr_key is not None and primary_asr_key not in rover_hyps:
                    primary_asr_key = sorted(rover_hyps.keys())[0] if rover_hyps else None

                rover = rover_combine(rover_hyps)
                t_rover1 = time.perf_counter()

                # 5.5) Audio-grounded fit scoring (optional): choose best candidate by CTC loss.
                t_ctc0 = time.perf_counter()
                ctc_info = None
                final_text = rover.text
                final_source = "rover"
                if router_key and router_key in rover_hyps and bool(thresholds.get("final_prefer_router", False)):
                    final_text = rover_hyps[router_key]
                    final_source = f"asr:{router_key}"
                elif bool(thresholds.get("final_prefer_transformers", False)):
                    tf_keys = [k for k in rover_hyps.keys() if k.startswith("transformers:")]
                    if tf_keys:
                        preferred = tf_keys[0]
                        final_text = rover_hyps[preferred]
                        final_source = f"asr:{preferred}"
                if ctc_scorer is not None and rank_by_ctc is not None:
                    candidates = {"rover": rover.text, **rover_hyps}
                    ctc_info = rank_by_ctc(ctc_scorer, wav_path, candidates=candidates, lang=route_lang)
                    best_key = ctc_info.get("best_key") if isinstance(ctc_info, dict) else None
                    if best_key and best_key in candidates:
                        final_text = candidates[best_key]
                        final_source = "rover" if best_key == "rover" else f"asr:{best_key}"
                t_ctc1 = time.perf_counter()

                # 6) Verify
                t_ver0 = time.perf_counter()
                asr_summaries = {k: _asr_summary(v) for k, v in asr_results.items()}
                verification = verify_segment(
                    qc=asdict(qc),
                    lid=lid,
                    final_text=final_text,
                    rover={
                        "text": rover.text,
                        "disagreement_rate": rover.disagreement_rate,
                        "skeleton_key": rover.skeleton_key,
                    },
                    asr_summaries=asr_summaries,
                    primary_asr_key=primary_asr_key,
                    ctc=ctc_info,
                    thresholds=thresholds,
                )
                t_ver1 = time.perf_counter()

                record = {
                    "schema_version": 1,
                    "segment_id": segment_id,
                    "base_id": ident.base_id,
                    "relpath": ident.relpath,
                    "uid": ident.uid,
                    "src_path": str(src),
                    "wav_path": str(wav_path),
                    "src_ffprobe": asdict(src_info),
                    "ffprobe": asdict(info),
                    "qc": asdict(qc),
                    "lid": (
                        None
                        if lid is None
                        else {
                            "model": lid.model,
                            "predictions": [{"code": p.code, "prob": p.prob} for p in lid.predictions],
                        }
                    ),
                    "route": {"language_hint": route_lang, "language_hint_source": route_lang_source},
                    "asr": {
                        k: {
                            "model": v.model,
                            "text": v.text,
                            "language": v.language,
                            "language_probability": v.language_probability,
                            "segments": [asdict(s) for s in v.segments],
                            "meta": v.meta,
                        }
                        for k, v in asr_results.items()
                    },
                    "asr_summary": asr_summaries,
                    "asr_errors": asr_errors,
                    "ensemble": {
                        "unique_models": ensemble_models,
                        "unique_backends": ensemble_backends,
                        "warnings": ensemble_warnings,
                    },
                    "normalized": norm_hyps,
                    "rover": {
                        "text": rover.text,
                        "skeleton_key": rover.skeleton_key,
                        "disagreement_rate": rover.disagreement_rate,
                        "inputs": sorted(rover_hyps.keys()),
                        "filtered_out": sorted(set(rover_filtered_out)),
                        "disagreements": [
                            {"kind": d.kind, "index": d.index, "chosen": d.chosen, "options": d.options}
                            for d in rover.disagreements
                        ],
                    },
                    "ctc": ctc_info,
                    "final": {"text": final_text, "source": final_source},
                    "verification": {
                        "decision": verification.decision,
                        "reasons": verification.reasons,
                        "scores": verification.scores,
                    },
                    "config": {
                        "trim_silence": trim_silence,
                        "silence_threshold_db": silence_threshold_db,
                        "min_silence_sec": min_silence_sec,
                        "gemini_model": gemini_model,
                        "gemini_hard_only": gemini_hard_only,
                        "ctc_model": ctc_model,
                        "mms_asr_model": mms_asr_model,
                        "indicconformer_default_model": indicconformer_default_model,
                    },
                    "timing": {
                        "preprocess_sec": t_pre1 - t_pre0,
                        "lid_sec": t_lid1 - t_lid0,
                        "asr_sec": t_asr1 - t_asr0,
                        "normalize_sec": t_norm1 - t_norm0,
                        "rover_sec": t_rover1 - t_rover0,
                        "ctc_sec": t_ctc1 - t_ctc0,
                        "verify_sec": t_ver1 - t_ver0,
                        "total_sec": time.perf_counter() - t0,
                    },
                }

                f.write(json.dumps(record, ensure_ascii=False) + "\n")
                decision = verification.decision
                counts[decision] = counts.get(decision, 0) + 1
                timings_total.append(record["timing"]["total_sec"])

            except Exception as e:  # pragma: no cover
                counts["error"] += 1
                f.write(
                    json.dumps(
                        {"segment_id": segment_id, "src_path": str(src), "error": str(e)}, ensure_ascii=False
                    )
                    + "\n"
                )

    summary = {
        "segments_total": len(audio_files),
        "counts": counts,
        "avg_total_sec": (sum(timings_total) / len(timings_total)) if timings_total else None,
        "p50_total_sec": _percentile(timings_total, 0.50),
        "p90_total_sec": _percentile(timings_total, 0.90),
    }
    summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    return results_path

def _percentile(xs: list[float], q: float) -> float | None:
    if not xs:
        return None
    xs_sorted = sorted(xs)
    idx = int(round((len(xs_sorted) - 1) * q))
    idx = max(0, min(len(xs_sorted) - 1, idx))
    return float(xs_sorted[idx])


**Assessing integration of new audio LLMs**

The pipeline currently uses ASR backends without LLM audio support, so integrating new models like Voxtral or Gemma3n likely requires adding a new backend or wrapper. Since vLLM integration is missing and the user wants to avoid transformers, running inference externally with vLLM or other engines and evaluating outputs separately could be easiest. Next, I’ll research Voxtral’s model type and compatibility with vLLM or audio-specific LLM setups.


**Planning to check huggingface_hub availability**

I'll verify if huggingface_hub is installed by trying to import it via a Python one-liner before deciding whether to install it for listing or searching models.


python - <<'PY'
import importlib.util
print('huggingface_hub', importlib.util.find_spec('huggingface_hub') is not None)
print('vllm', importlib.util.find_spec('vllm') is not None)
PY


Output:
Exit code: 127
Wall time: 0 seconds
Output:
/bin/bash: line 1: python: command not found


**Checking for Python3 availability**


python3 - <<'PY'
import importlib.util
print('huggingface_hub', importlib.util.find_spec('huggingface_hub') is not None)
print('vllm', importlib.util.find_spec('vllm') is not None)
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
huggingface_hub True
vllm True


**Confirming vllm installation and support**


python3 - <<'PY'
import vllm, sys
print('vllm', vllm.__version__)
PY


Output:
Exit code: 0
Wall time: 5.1 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
vllm 0.13.0


**Researching audio model availability**

I'm gathering potential audio-capable open-source models like "Voxtral" and variants of "Gemma3n" by planning to search Hugging Face with huggingface_hub to verify if vLLM or related models support audio input.


python3 - <<'PY'
from huggingface_hub import HfApi
api = HfApi()
for name in ["voxtral", "gemma3n", "gemma-3n", "gemma 3n", "gemma-3", "gemma-3n-it"]:
    models = api.list_models(search=name, limit=5)
    print("\nsearch", name)
    for m in models:
        print("-", m.modelId)
PY


Output:
Exit code: 0
Wall time: 0.9 seconds
Output:

search voxtral
- mistralai/Voxtral-Mini-3B-2507
- bartowski/mistralai_Voxtral-Mini-3B-2507-GGUF
- bartowski/mistralai_Voxtral-Small-24B-2507-GGUF
- elyx/voxtral-overfit
- mistralai/Voxtral-Small-24B-2507

search gemma3n
- google/gemma-3n-E2B-it-litert-lm
- google/gemma-3n-E4B-it-litert-lm
- google/gemma-3n-E2B-it
- google/gemma-3n-E4B-it
- google/gemma-3n-E4B

search gemma-3n
- google/gemma-3n-E2B-it-litert-lm
- google/gemma-3n-E4B-it-litert-lm
- google/gemma-3n-E2B-it
- google/gemma-3n-E4B-it
- google/gemma-3n-E4B

search gemma 3n
- google/gemma-3n-E2B-it-litert-lm
- google/gemma-3n-E4B-it-litert-lm
- google/gemma-3n-E2B-it
- google/gemma-3n-E4B-it
- google/gemma-3n-E4B

search gemma-3
- DreamFast/gemma-3-12b-it-heretic
- unsloth/gemma-3-12b-it-bnb-4bit
- google/gemma-3-12b-it-qat-q4_0-unquantized
- google/gemma-3-4b-it
- google/gemma-3-27b-it

search gemma-3n-it
- google/gemma-3n-E2B-it-litert-lm
- google/gemma-3n-E4B-it-litert-lm
- google/gemma-3n-E2B-it
- google/gemma-3n-E4B-it
- unsloth/gemma-3n-E2B-it-GGUF


Summary: auto


**Checking model type and metadata**

I'll inspect the Voxtral models' metadata and config to confirm if they are audio models or speech-to-text by retrieving their architectures and model types from the Hugging Face hub.


python3 - <<'PY'
from huggingface_hub import HfApi
api = HfApi()
for model_id in ["mistralai/Voxtral-Mini-3B-2507", "mistralai/Voxtral-Small-24B-2507", "google/gemma-3n-E2B-it", "google/gemma-3n-E4B-it"]:
    info = api.model_info(model_id)
    print("\n", model_id)
    print("tags:", info.tags)
    print("pipeline_tag:", info.pipeline_tag)
    print("library_name:", info.library_name)
    print("config:", list(info.config.keys()) if info.config else None)
PY


Output:
Exit code: 0
Wall time: 0.7 seconds
Output:

 mistralai/Voxtral-Mini-3B-2507
tags: ['mistral-common', 'safetensors', 'voxtral', 'vllm', 'en', 'fr', 'de', 'es', 'it', 'pt', 'nl', 'hi', 'arxiv:2507.13264', 'license:apache-2.0', 'region:us']
pipeline_tag: None
library_name: mistral-common
config: ['architectures', 'model_type']

 mistralai/Voxtral-Small-24B-2507
tags: ['vllm', 'safetensors', 'voxtral', 'audio-text-to-text', 'en', 'fr', 'de', 'es', 'it', 'pt', 'nl', 'hi', 'arxiv:2507.13264', 'base_model:mistralai/Mistral-Small-24B-Base-2501', 'base_model:finetune:mistralai/Mistral-Small-24B-Base-2501', 'license:apache-2.0', 'region:us']
pipeline_tag: audio-text-to-text
library_name: vllm
config: ['architectures', 'model_type']

 google/gemma-3n-E2B-it
tags: ['transformers', 'safetensors', 'gemma3n', 'image-to-text', 'automatic-speech-recognition', 'automatic-speech-translation', 'audio-text-to-text', 'video-text-to-text', 'image-text-to-text', 'conversational', 'arxiv:1905.07830', 'arxiv:1905.10044', 'arxiv:1911.11641', 'arxiv:1904.09728', 'arxiv:1705.03551', 'arxiv:1911.01547', 'arxiv:1907.10641', 'arxiv:1903.00161', 'arxiv:2210.03057', 'arxiv:2502.12404', 'arxiv:2411.19799', 'arxiv:2009.03300', 'arxiv:2502.21228', 'arxiv:2311.12022', 'arxiv:2403.07974', 'arxiv:2108.07732', 'arxiv:2107.03374', 'base_model:google/gemma-3n-E4B-it', 'base_model:finetune:google/gemma-3n-E4B-it', 'license:gemma', 'endpoints_compatible', 'region:us']
pipeline_tag: image-text-to-text
library_name: transformers
config: ['architectures', 'model_type', 'tokenizer_config', 'chat_template_jinja']

 google/gemma-3n-E4B-it
tags: ['transformers', 'safetensors', 'gemma3n', 'image-to-text', 'automatic-speech-recognition', 'automatic-speech-translation', 'audio-text-to-text', 'video-text-to-text', 'image-text-to-text', 'conversational', 'arxiv:1905.07830', 'arxiv:1905.10044', 'arxiv:1911.11641', 'arxiv:1904.09728', 'arxiv:1705.03551', 'arxiv:1911.01547', 'arxiv:1907.10641', 'arxiv:1903.00161', 'arxiv:2210.03057', 'arxiv:2502.12404', 'arxiv:2411.19799', 'arxiv:2009.03300', 'arxiv:2502.21228', 'arxiv:2311.12022', 'arxiv:2403.07974', 'arxiv:2108.07732', 'arxiv:2107.03374', 'base_model:google/gemma-3n-E4B', 'base_model:finetune:google/gemma-3n-E4B', 'license:gemma', 'endpoints_compatible', 'region:us']
pipeline_tag: image-text-to-text
library_name: transformers
config: ['architectures', 'model_type', 'tokenizer_config', 'chat_template_jinja']


**Checking vLLM audio and GPU support**

I'm investigating whether vLLM supports audio or multimodal models like gemma3n and voxtral, noting potential multimodal features in vLLM 0.13.0. Next, I'll verify GPU availability with nvidia-smi to enable running these models.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-ada3fc23-5709-2bd4-dd15-24f3b0c3cd2d)


**Planning benchmark evaluation**

The user likely wants sequential evaluation of mini 3B and small 24B on benchmark data, specifically using datasets like `data/bench/maya_hf_hin_v2`. Next, I'll check benchmark directories and the CLI commands in `maya_transcribe/cli.py` to learn how to run and compute WER on these benchmarks.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
20:bench_app = typer.Typer(add_completion=False, no_args_is_help=True, help="Benchmarking utilities.")
21:app.add_typer(bench_app, name="bench")
330:@bench_app.command("build")
331:def bench_build(
332:    out: Path = typer.Option(Path("data/bench/maya_hf_v0"), help="Output benchmark corpus directory."),
342:    Build a local benchmark corpus from Hugging Face datasets.
344:    from maya_transcribe.bench import build_benchmark_corpus
346:    manifest = build_benchmark_corpus(
359:@bench_app.command("run")
360:def bench_run(
361:    bench_dir: Path = typer.Option(Path("data/bench/maya_hf_v0"), exists=True, file_okay=False, help="Benchmark corpus directory."),
362:    out: Path | None = typer.Option(None, help="Output directory (defaults under outputs/bench/)."),
388:        False, help="Trim leading/trailing silence (ffmpeg silenceremove). Off by default for benchmarking."
397:    Run the full pipeline on a local benchmark corpus and write eval.json.
404:    audio_dir = bench_dir / "audio"
405:    refs_path = bench_dir / "refs.jsonl"
413:        out = Path("outputs/bench") / bench_dir.name / f"run_{ts}"
478:@bench_app.command("push")
479:def bench_push(
480:    bench_dir: Path = typer.Option(Path("data/bench/maya_hf_v0"), exists=True, file_okay=False, help="Benchmark corpus directory."),
487:    Push a local benchmark corpus to the Hugging Face Hub as a (private) dataset.
489:    from maya_transcribe.bench import push_benchmark_to_hub
491:    repo = push_benchmark_to_hub(bench_dir=bench_dir, repo_id=repo_id, private=private, include_audio=include_audio, split=split)


sed -n '340,460p' maya_transcribe/cli.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
) -> None:
    """
    Build a local benchmark corpus from Hugging Face datasets.
    """
    from maya_transcribe.bench import build_benchmark_corpus

    manifest = build_benchmark_corpus(
        out_dir=out,
        samples_per_lang=samples_per_lang,
        seed=seed,
        shuffle=shuffle,
        langs=lang or None,
        min_duration_sec=min_duration_sec,
        max_duration_sec=max_duration_sec,
        force=force,
    )
    typer.echo(json.dumps({"wrote": str(out), "total_rows": manifest.get("total_rows"), "counts": manifest.get("counts")}, indent=2))

@bench_app.command("run")
def bench_run(
    bench_dir: Path = typer.Option(Path("data/bench/maya_hf_v0"), exists=True, file_okay=False, help="Benchmark corpus directory."),
    out: Path | None = typer.Option(None, help="Output directory (defaults under outputs/bench/)."),
    whisper_model: str = typer.Option("large-v3", help="Whisper model size/path for faster-whisper."),
    lid_model_id: str = typer.Option("facebook/mms-lid-256", help="HuggingFace MMS-LID model id."),
    require_lid_backend: bool = typer.Option(True, help="Fail if MMS-LID backend can't initialize."),
    transformers_whisper_model: list[str] = typer.Option(
        [], help="Extra whisper-family HF models to run via transformers (repeatable)."
    ),
    transformers_whisper_model_map: Path | None = typer.Option(
        None, help="JSON map {lang_code: hf_model_id} for per-language Whisper-family HF models (transformers)."
    ),
    # MMS ASR for CTC-based model diversity (per master.md plan).
    mms_asr_model: str | None = typer.Option(
        None, help="Optional MMS ASR model id (e.g., facebook/mms-1b-all) for CTC-based diversity."
    ),
    # IndicConformer for Conformer architecture diversity (per master.md plan).
    indicconformer_model_map: Path | None = typer.Option(
        None, help="JSON map {lang_code: nemo_model_name} for per-language IndicConformer models."
    ),
    indicconformer_default_model: str | None = typer.Option(
        None, help="Default IndicConformer NeMo model name for architecture diversity."
    ),
    ctc_model: str | None = typer.Option(None, help="Optional HF CTC model id for audio-grounded fit scoring."),
    ctc_model_map: Path | None = typer.Option(None, help="JSON map {lang_code: hf_model_id} for CTC scoring."),
    require_ctc: bool = typer.Option(True, help="When CTC is enabled, require a CTC score to accept."),
    min_ctc_margin: float = typer.Option(0.05, help="Min (second-best - best) CTC loss margin to auto-accept."),
    trim_silence: bool = typer.Option(
        False, help="Trim leading/trailing silence (ffmpeg silenceremove). Off by default for benchmarking."
    ),
    silence_threshold_db: float = typer.Option(-40.0, help="Silence threshold in dB for trimming."),
    min_silence_sec: float = typer.Option(0.15, help="Min silence duration for trimming."),
    limit: int | None = typer.Option(None, help="Process only the first N segments."),
    thresholds_json: Path | None = typer.Option(None, help="Optional JSON file of threshold overrides (verify + routing)."),
    hypothesis: str = typer.Option("final", help='Hypothesis to evaluate: "final", "rover", or "asr:<key>".'),
) -> None:
    """
    Run the full pipeline on a local benchmark corpus and write eval.json.
    """
    from datetime import datetime

    from maya_transcribe.eval import evaluate
    from maya_transcribe.pipeline import run_pipeline

    audio_dir = bench_dir / "audio"
    refs_path = bench_dir / "refs.jsonl"
    if not audio_dir.exists():
        raise typer.BadParameter(f"Missing: {audio_dir}")
    if not refs_path.exists():
        raise typer.BadParameter(f"Missing: {refs_path}")

    if out is None:
        ts = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
        out = Path("outputs/bench") / bench_dir.name / f"run_{ts}"

    ctc_map = None
    if ctc_model_map is not None:
        ctc_map = json.loads(ctc_model_map.read_text(encoding="utf-8"))
        if not isinstance(ctc_map, dict):
            raise typer.BadParameter("ctc_model_map must be a JSON object {lang_code: model_id}")

    tf_map = None
    if transformers_whisper_model_map is not None:
        tf_map = json.loads(transformers_whisper_model_map.read_text(encoding="utf-8"))
        if not isinstance(tf_map, dict):
            raise typer.BadParameter("transformers_whisper_model_map must be a JSON object {lang_code: model_id}")

    ic_map = None
    if indicconformer_model_map is not None:
        ic_map = json.loads(indicconformer_model_map.read_text(encoding="utf-8"))
        if not isinstance(ic_map, dict):
            raise typer.BadParameter("indicconformer_model_map must be a JSON object {lang_code: nemo_model_name}")

    thresholds: dict[str, object] = {"route_lang_from_path": True, "route_lang_prefer_path": True}
    if thresholds_json is not None:
        loaded = json.loads(thresholds_json.read_text(encoding="utf-8"))
        if not isinstance(loaded, dict):
            raise typer.BadParameter("thresholds_json must be a JSON object")
        thresholds.update(loaded)
    if ctc_model or ctc_map:
        thresholds["require_ctc"] = require_ctc
        thresholds["min_ctc_margin"] = float(min_ctc_margin)

    results_path = run_pipeline(
        segments_dir=audio_dir,
        out_dir=out,
        lid_model_id=lid_model_id,
        require_lid_backend=require_lid_backend,
        whisper_model=whisper_model,
        extra_transformers_whisper_models=transformers_whisper_model,
        transformers_whisper_model_map=tf_map,
        mms_asr_model=mms_asr_model,
        indicconformer_model_map=ic_map,
        indicconformer_default_model=indicconformer_default_model,
        ctc_model=ctc_model,
        ctc_model_map=ctc_map,
        trim_silence=trim_silence,
        silence_threshold_db=silence_threshold_db,
        min_silence_sec=min_silence_sec,
        limit=limit,
        thresholds=thresholds,


sed -n '460,560p' maya_transcribe/cli.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
        thresholds=thresholds,
    )

    eval_path = out / "eval.json"
    summary = evaluate(
        results_path=results_path,
        refs_path=refs_path,
        out_path=eval_path,
        hypothesis=hypothesis,
        oracle=True,
        language_source="refs",
    )
    typer.echo(f"Wrote: {results_path}")
    typer.echo(f"Wrote: {out / 'summary.json'}")
    typer.echo(f"Wrote: {eval_path}")
    typer.echo(json.dumps(summary, ensure_ascii=False, indent=2))

@bench_app.command("push")
def bench_push(
    bench_dir: Path = typer.Option(Path("data/bench/maya_hf_v0"), exists=True, file_okay=False, help="Benchmark corpus directory."),
    repo_id: str = typer.Option(..., help="Hugging Face dataset repo id (e.g., org/name)."),
    private: bool = typer.Option(True, help="Create/push as a private dataset repo."),
    include_audio: bool = typer.Option(False, help="Include audio bytes in the pushed dataset (check dataset licenses!)."),
    split: str = typer.Option("test", help="Split name to use on the Hub."),
) -> None:
    """
    Push a local benchmark corpus to the Hugging Face Hub as a (private) dataset.
    """
    from maya_transcribe.bench import push_benchmark_to_hub

    repo = push_benchmark_to_hub(bench_dir=bench_dir, repo_id=repo_id, private=private, include_audio=include_audio, split=split)
    typer.echo(f"Pushed: {repo}")

if __name__ == "__main__":
    app()


**Inspecting new ASR backend usage**

I'm confirming that for Voxtral we'll avoid Transformers and focus on vLLM or Mistral's inference via `mistral-common`. Next, I'll fetch and review the Voxtral README from Huggingface to clarify usage and integration steps.


python3 - <<'PY'
from huggingface_hub import hf_hub_download
for model_id in ["mistralai/Voxtral-Mini-3B-2507", "mistralai/Voxtral-Small-24B-2507", "google/gemma-3n-E2B-it", "google/gemma-3n-E4B-it"]:
    try:
        path = hf_hub_download(repo_id=model_id, filename="README.md")
        print("\n===", model_id, "===")
        with open(path, "r", encoding="utf-8") as f:
            # print first 80 lines
            for i in range(80):
                line = f.readline()
                if not line:
                    break
                print(line.rstrip())
    except Exception as e:
        print("\n===", model_id, "===")
        print("README fetch failed:", e)
PY


Output:
Exit code: 0
Wall time: 1.8 seconds
Output:

=== mistralai/Voxtral-Mini-3B-2507 ===
---
library_name: mistral-common
language:
- en
- fr
- de
- es
- it
- pt
- nl
- hi
license: apache-2.0
inference: false
extra_gated_description: >-
  If you want to learn more about how we process your personal data, please read
  our Privacy Policy.
tags:
- vllm
---
# Voxtral Mini 1.0 (3B) - 2507

Voxtral Mini is an enhancement of [Ministral 3B](https://mistral.ai/news/ministraux), incorporating state-of-the-art audio input capabilities while retaining best-in-class text performance. It excels at speech transcription, translation and audio understanding.

Learn more about Voxtral in our blog post [here](https://mistral.ai/news/voxtral) and our [research paper](https://arxiv.org/abs/2507.13264).

## Key Features

Voxtral builds upon Ministral-3B with powerful audio understanding capabilities.
- **Dedicated transcription mode**: Voxtral can operate in a pure speech transcription mode to maximize performance. By default, Voxtral automatically predicts the source audio language and transcribes the text accordingly
- **Long-form context**: With a 32k token context length, Voxtral handles audios up to 30 minutes for transcription, or 40 minutes for understanding
- **Built-in Q&A and summarization**: Supports asking questions directly through audio. Analyze audio and generate structured summaries without the need for separate ASR and language models
- **Natively multilingual**: Automatic language detection and state-of-the-art performance in the world’s most widely used languages (English, Spanish, French, Portuguese, Hindi, German, Dutch, Italian)
- **Function-calling straight from voice**: Enables direct triggering of backend functions, workflows, or API calls based on spoken user intents
- **Highly capable at text**: Retains the text understanding capabilities of its language model backbone, Ministral-3B

## Benchmark Results

### Audio

Average word error rate (WER) over the FLEURS, Mozilla Common Voice and Multilingual LibriSpeech benchmarks:

![image/png](https://cdn-uploads.huggingface.co/production/uploads/64161701107962562e9b1006/puASxtajF1lDeGYPrRK5y.png)

### Text

![image/png](https://cdn-uploads.huggingface.co/production/uploads/5dfcb1aada6d0311fd3d5448/iH9V8JVtMoaGlqJd6FIri.png)

## Usage

The model can be used with the following frameworks;
- [`vllm (recommended)`](https://github.com/vllm-project/vllm): See [here](#vllm-recommended)
- [`Transformers` 🤗](https://github.com/huggingface/transformers): See [here](#transformers-🤗)

**Notes**:

- `temperature=0.2` and `top_p=0.95` for chat completion (*e.g. Audio Understanding*) and `temperature=0.0` for transcription
- Multiple audios per message and multiple user turns with audio are supported
- System prompts are not yet supported

### vLLM (recommended)

We recommend using this model with [vLLM](https://github.com/vllm-project/vllm).

#### Installation

Make sure to install vllm >= 0.10.0, we recommend using `uv`:

```
uv pip install -U "vllm[audio]" --system
```

Doing so should automatically install [`mistral_common >= 1.8.1`](https://github.com/mistralai/mistral-common/releases/tag/v1.8.1).

To check:
```
python -c "import mistral_common; print(mistral_common.__version__)"
```

#### Offline

=== mistralai/Voxtral-Small-24B-2507 ===
---
library_name: vllm
language:
- en
- fr
- de
- es
- it
- pt
- nl
- hi
license: apache-2.0
inference: false
base_model:
- mistralai/Mistral-Small-24B-Base-2501
extra_gated_description: >-
  If you want to learn more about how we process your personal data, please read
  our Privacy Policy.
tags:
- vllm
pipeline_tag: audio-text-to-text
---

# Voxtral Small 1.0 (24B) - 2507

Voxtral Small is an enhancement of [Mistral Small 3](https://huggingface.co/mistralai/Mistral-Small-24B-Base-2501), incorporating state-of-the-art audio input capabilities while retaining best-in-class text performance. It excels at speech transcription, translation and audio understanding.

Learn more about Voxtral in our blog post [here](https://mistral.ai/news/voxtral) and our [research paper](https://arxiv.org/abs/2507.13264).

## Key Features

Voxtral builds upon Mistral Small 3 with powerful audio understanding capabilities.
- **Dedicated transcription mode**: Voxtral can operate in a pure speech transcription mode to maximize performance. By default, Voxtral automatically predicts the source audio language and transcribes the text accordingly
- **Long-form context**: With a 32k token context length, Voxtral handles audios up to 30 minutes for transcription, or 40 minutes for understanding
- **Built-in Q&A and summarization**: Supports asking questions directly through audio. Analyze audio and generate structured summaries without the need for separate ASR and language models
- **Natively multilingual**: Automatic language detection and state-of-the-art performance in the world’s most widely used languages (English, Spanish, French, Portuguese, Hindi, German, Dutch, Italian)
- **Function-calling straight from voice**: Enables direct triggering of backend functions, workflows, or API calls based on spoken user intents
- **Highly capable at text**: Retains the text understanding capabilities of its language model backbone, Mistral Small 3.1

## Benchmark Results

### Audio

Average word error rate (WER) over the FLEURS, Mozilla Common Voice and Multilingual LibriSpeech benchmarks:

![image/png](https://cdn-uploads.huggingface.co/production/uploads/64161701107962562e9b1006/puASxtajF1lDeGYPrRK5y.png)

### Text

![image/png](https://cdn-uploads.huggingface.co/production/uploads/5dfcb1aada6d0311fd3d5448/uDg3hKDwJowsNuj-yyt2T.png)

## Usage

The model can be used with the following frameworks;
- [`vllm (recommended)`](https://github.com/vllm-project/vllm): See [here](#vllm-recommended)
- [`Transformers` 🤗](https://github.com/huggingface/transformers): See [here](#transformers-🤗)

**Notes**:

- `temperature=0.2` and `top_p=0.95` for chat completion (*e.g. Audio Understanding*) and `temperature=0.0` for transcription
- Multiple audios per message and multiple user turns with audio are supported
- Function calling is supported
- System prompts are not yet supported

### vLLM (recommended)

We recommend using this model with [vLLM](https://github.com/vllm-project/vllm).

#### Installation

Make sure to install vllm >= `0.10.0`, we recommend using uv

```
uv pip install -U "vllm[audio]" --system
```

Doing so should automatically install [`mistral_common >= 1.8.1`](https://github.com/mistralai/mistral-common/releases/tag/v1.8.1).

=== google/gemma-3n-E2B-it ===
---
license: gemma
library_name: transformers
pipeline_tag: image-text-to-text
extra_gated_heading: Access Gemma on Hugging Face
extra_gated_prompt: To access Gemma on Hugging Face, you’re required to review and
  agree to Google’s usage license. To do this, please ensure you’re logged in to Hugging
  Face and click below. Requests are processed immediately.
extra_gated_button_content: Acknowledge license
base_model: google/gemma-3n-E4B-it
tags:
- automatic-speech-recognition
- automatic-speech-translation
- audio-text-to-text
- video-text-to-text
---

> [!Note]
> This repository corresponds to the launch version of Gemma 3n E2B IT (Instruct), to be used with Hugging Face `transformers`,
> supporting text, audio, and vision (image and video) inputs.
>
> Gemma 3n models have multiple architecture innovations:
>  * They are available in two sizes based on [effective parameters](https://ai.google.dev/gemma/docs/gemma-3n#parameters). While the raw parameter count of this model is 6B, the architecture design allows the model to be run with a memory footprint comparable to a traditional 2B model by offloading low-utilization matrices from the accelerator.
>  * They use a MatFormer architecture that allows nesting sub-models within the [E4B model](https://huggingface.co/google/gemma-3n-E4B-it). We provide one sub-model (this model repository), or you can access a spectrum of custom-sized models using the [Mix-and-Match method](https://goo.gle/gemma3n-matformer-lab).
>
> Learn more about these techniques in the [technical blog post](https://developers.googleblog.com/en/introducing-gemma-3n-developer-guide)
> and the [Gemma documentation](https://ai.google.dev/gemma/docs/gemma-3n).

# Gemma 3n model card

**Model Page**: [Gemma 3n](https://ai.google.dev/gemma/docs/gemma-3n)

**Resources and Technical Documentation**:

-   [Responsible Generative AI Toolkit](https://ai.google.dev/responsible)
-   [Gemma on Kaggle](https://www.kaggle.com/models/google/gemma-3n)
-   [Gemma on HuggingFace](https://huggingface.co/collections/google/gemma-3n-685065323f5984ef315c93f4)
-   [Gemma on Vertex Model Garden](https://console.cloud.google.com/vertex-ai/publishers/google/model-garden/gemma3n)

**Terms of Use**: [Terms](https://ai.google.dev/gemma/terms)\
**Authors**: Google DeepMind

## Model Information

Summary description and brief definition of inputs and outputs.

### Description

Gemma is a family of lightweight, state-of-the-art open models from Google,
built from the same research and technology used to create the Gemini models.
Gemma 3n models are designed for efficient execution on low-resource devices.
They are capable of multimodal input, handling text, image, video, and audio
input, and generating text outputs, with open weights for pre-trained and
instruction-tuned variants. These models were trained with data in over 140
spoken languages.

Gemma 3n models use selective parameter activation technology to reduce resource
requirements. This technique allows the models to operate at an effective size
of 2B and 4B parameters, which is lower than the total number of parameters they
contain. For more information on Gemma 3n's efficient parameter management
technology, see the
[Gemma 3n](https://ai.google.dev/gemma/docs/gemma-3n#parameters)
page.

### Inputs and outputs

-   **Input:**
    -   Text string, such as a question, a prompt, or a document to be
        summarized
    -   Images, normalized to 256x256, 512x512, or 768x768 resolution
        and encoded to 256 tokens each
    -   Audio data encoded to 6.25 tokens per second from a single channel
    -   Total input context of 32K tokens
-   **Output:**
    -   Generated text in response to the input, such as an answer to a
        question, analysis of image content, or a summary of a document
    -   Total output length up to 32K tokens, subtracting the request
        input tokens

=== google/gemma-3n-E4B-it ===
---
license: gemma
library_name: transformers
pipeline_tag: image-text-to-text
extra_gated_heading: Access Gemma on Hugging Face
extra_gated_prompt: >-
  To access Gemma on Hugging Face, you’re required to review and agree to
  Google’s usage license. To do this, please ensure you’re logged in to Hugging
  Face and click below. Requests are processed immediately.
extra_gated_button_content: Acknowledge license
base_model: google/gemma-3n-E4B
tags:
- automatic-speech-recognition
- automatic-speech-translation
- audio-text-to-text
- video-text-to-text
---

> [!Note]
> This repository corresponds to the launch version of Gemma 3n E4B IT (Instruct), to be used with Hugging Face `transformers`,
> supporting text, audio, and vision (image and video) inputs.
>
> Gemma 3n models have multiple architecture innovations:
>  * They are available in two sizes based on [effective parameters](https://ai.google.dev/gemma/docs/gemma-3n#parameters). While the raw parameter count of this model is 8B, the architecture design allows the model to be run with a memory footprint comparable to a traditional 4B model by offloading low-utilization matrices from the accelerator.
>  * They use a MatFormer architecture that allows nesting sub-models within the E4B model. We provide one sub-model (an [E2B](https://huggingface.co/google/gemma-3n-E2B-it)), or you can access a spectrum of custom-sized models using the [Mix-and-Match method](https://goo.gle/gemma3n-matformer-lab).
>
> Learn more about these techniques in the [technical blog post](https://developers.googleblog.com/en/introducing-gemma-3n-developer-guide)
> and the [Gemma documentation](https://ai.google.dev/gemma/docs/gemma-3n).

# Gemma 3n model card

**Model Page**: [Gemma 3n](https://ai.google.dev/gemma/docs/gemma-3n)

**Resources and Technical Documentation**:

-   [Responsible Generative AI Toolkit](https://ai.google.dev/responsible)
-   [Gemma on Kaggle](https://www.kaggle.com/models/google/gemma-3n)
-   [Gemma on HuggingFace](https://huggingface.co/collections/google/gemma-3n-685065323f5984ef315c93f4)
-   [Gemma on Vertex Model Garden](https://console.cloud.google.com/vertex-ai/publishers/google/model-garden/gemma3n)

**Terms of Use**: [Terms](https://ai.google.dev/gemma/terms)\
**Authors**: Google DeepMind

## Model Information

Summary description and brief definition of inputs and outputs.

### Description

Gemma is a family of lightweight, state-of-the-art open models from Google,
built from the same research and technology used to create the Gemini models.
Gemma 3n models are designed for efficient execution on low-resource devices.
They are capable of multimodal input, handling text, image, video, and audio
input, and generating text outputs, with open weights for pre-trained and
instruction-tuned variants. These models were trained with data in over 140
spoken languages.

Gemma 3n models use selective parameter activation technology to reduce resource
requirements. This technique allows the models to operate at an effective size
of 2B and 4B parameters, which is lower than the total number of parameters they
contain. For more information on Gemma 3n's efficient parameter management
technology, see the
[Gemma 3n](https://ai.google.dev/gemma/docs/gemma-3n#parameters)
page.

### Inputs and outputs

-   **Input:**
    -   Text string, such as a question, a prompt, or a document to be
        summarized
    -   Images, normalized to 256x256, 512x512, or 768x768 resolution
        and encoded to 256 tokens each
    -   Audio data encoded to 6.25 tokens per second from a single channel
    -   Total input context of 32K tokens
-   **Output:**
    -   Generated text in response to the input, such as an answer to a
        question, analysis of image content, or a summary of a document
    -   Total output length up to 32K tokens, subtracting the request
        input tokens


**Assessing vLLM support for multimodal Gemma model**

I'm examining whether vLLM supports Gemma 3n's multimodal audio-text input, since the model card recommends transformers but the user prefers vLLM; next, I'll verify vLLM's audio capabilities and multimodal model support to confirm feasibility.


python3 - <<'PY'
import pkgutil
import vllm
import vllm.model_executor.models as models
names = [m.name for m in pkgutil.iter_modules(models.__path__)]
print('models:', sorted(names)[:60])
print('has gemma3n', any('gemma' in n for n in names))
print('has mistral', any('mistral' in n for n in names))
print('has audio', any('audio' in n for n in names))
PY


Output:
Exit code: 124
Wall time: 10.1 seconds
Output:
command timed out after 10098 milliseconds
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-14 22:57:31.097664: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
models: ['adapters', 'afmoe', 'aimv2', 'apertus', 'arcee', 'arctic', 'aria', 'audioflamingo3', 'aya_vision', 'bagel', 'baichuan', 'bailing_moe', 'bamba', 'bee', 'bert', 'bert_with_rope', 'blip', 'blip2', 'bloom', 'chameleon', 'chatglm', 'clip', 'cohere2_vision', 'commandr', 'config', 'dbrx', 'deepencoder', 'deepseek_eagle', 'deepseek_mtp', 'deepseek_ocr', 'deepseek_v2', 'deepseek_vl2', 'dots1', 'dots_ocr', 'ernie45', 'ernie45_moe', 'ernie45_vl', 'ernie45_vl_moe', 'ernie_mtp', 'exaone', 'exaone4', 'fairseq2_llama', 'falcon', 'falcon_h1', 'flex_olmo', 'fuyu', 'gemma', 'gemma2', 'gemma3', 'gemma3_mm', 'gemma3n', 'gemma3n_mm', 'glm', 'glm4', 'glm4_1v', 'glm4_moe', 'glm4_moe_mtp', 'glm4v', 'gpt2', 'gpt_bigcode']
has gemma3n True
has mistral True
has audio True


**Identifying vLLM models with multimodal support**

vLLM includes gemma3n and gemma3n_mm models, suggesting support for Gemma3n multimodal inputs. There's also mention of audioflamingo3, hinting at possible audio input support—next, I'll inspect the gemma3n_mm model code to clarify its input types and check for audio handling.


python3 - <<'PY'
import inspect
from vllm.model_executor.models import gemma3n_mm
print(inspect.getsource(gemma3n_mm)[:4000])
PY


Output:
Exit code: 0
Wall time: 11.2 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-14 22:57:46.792536: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, Optional, Union, cast

import numpy as np
import torch

from torch import nn
from transformers import AutoModel, BatchFeature
from transformers.models.gemma3n import (
    Gemma3nAudioConfig,
    Gemma3nAudioFeatureExtractor,
    Gemma3nConfig,
    Gemma3nProcessor,
    Gemma3nTextConfig,
    Gemma3nVisionConfig,
)
from transformers.models.siglip import SiglipImageProcessorFast

from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs.data import PromptType
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import RowParallelLinear
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.gemma3n import Gemma3nForCausalLM
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.whisper import ISO639_1_SUPPORTED_LANGS
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
    MultiModalDataDict,
    MultiModalFieldConfig,
    MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
    ImageProcessorItems,
    MultiModalDataItems,
    MultiModalDataParser,
)
from vllm.multimodal.processing import (
    BaseMultiModalProcessor,
    BaseProcessingInfo,
    MultiModalPromptUpdates,
    MultiModalPromptUpdatesApplyResult,
    PlaceholderFeaturesInfo,
    PromptReplacement,
    PromptUpdate,
    PromptUpdateDetails,
    replace_token_matches,
)
from vllm.multimodal.profiling import BaseDummyInputsBuilder
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape

from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsTranscription
from .utils import (
    AutoWeightsLoader,
    WeightsMapper,
    init_vllm_registered_model,
    maybe_prefix,
)

logger = init_logger(__name__)

# This should be based on model config but we hardcode them for now.
TOKENS_PER_IMAGE = 256
TOKENS_PER_AUDIO = 188

class Gemma3nImagePixelInputs(TensorSchema):
    """
    Dimensions:
        - bn: Batch size * number of images
        - c: Number of channels (3)
        - h: Height of each patch
        - w: Width of each patch
    """

    type: Literal["pixel_values"] = "pixel_values"
    pixel_values: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")]

class Gemma3nAudioInputs(TensorSchema):
    """
    Dimensions:
        - bn: Batch size * number of audios
        - s: seq_length
        - f: num_features
    """

    type: Literal["audio"] = "audio"
    input_features_padded: Annotated[torch.Tensor, TensorShape("bn", "s", "f")]
    input_features_mask: Annotated[torch.Tensor, TensorShape("bn", "s")]

Gemma3nImageInputs = Gemma3nImagePixelInputs

class Gemma3nProcessingInfo(BaseProcessingInfo):
    def get_hf_config(self):
        return self.ctx.get_hf_config(Gemma3nConfig)

    def get_hf_processor(self, **kwargs: object):
        return self.ctx.get_hf_processor(Gemma3nProcessor, **kwargs)

    def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
        return {"image": None, "audio": None}

    def get_max_tokens_per_item(
        self, seq_len: int, mm_counts: Mapping[str, int]
    ) -> Optional[Mapping[str, int]]:
        return {"image": TOKENS_PER_IMAGE, "audio": TOKENS_PER_AUDIO}

    def get_image_repl(
        self,
        *,
        image_width: int,
        image_height: int,
        processor: Optional[Gemma3nProcessor],
    ) -> str:
        """
        Get the replacement text for image tokens.

        For Gemma3n, this should return the full_image_sequence which includes
        BOI token, repeated image tokens, and EOI token.
        """
        i


**Assessing vLLM audio support**

Found that vLLM uses transformers' audio feature extractor for preprocessing, requiring `vllm[audio]` extras and likely dependencies like `mistral_common` and `librosa`; next step is verifying these are installed and checking if vLLM includes a model for "voxtral."


python3 - <<'PY'
import pkgutil
import vllm.model_executor.models as models
matches = []
for m in pkgutil.iter_modules(models.__path__):
    if 'vox' in m.name:
        matches.append(m.name)
print(matches)
PY


Output:
Exit code: 0
Wall time: 10.7 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-14 22:58:06.469208: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
['ultravox', 'voxtral']


**Identifying voxtral model in vLLM**


python3 - <<'PY'
import inspect
from vllm.model_executor.models import voxtral
print(inspect.getsource(voxtral)[:4000])
PY


Output:
Exit code: 0
Wall time: 11.9 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-14 22:58:19.888744: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import math
from collections.abc import Iterable, Mapping, Sequence
from functools import cached_property
from math import ceil
from typing import Literal, cast

import numpy as np
import regex as re
import torch
import torch.nn as nn
from mistral_common.audio import mel_filter_bank
from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
from mistral_common.protocol.transcription.request import TranscriptionRequest
from mistral_common.tokens.tokenizers.audio import Audio, AudioEncoder
from transformers import BatchFeature, TensorType, WhisperConfig
from transformers.tokenization_utils_base import TextInput

from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs.data import PromptType
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models import SupportsPP
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.whisper import WhisperEncoder
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
    MultiModalDataDict,
    MultiModalFieldConfig,
    MultiModalKwargsItems,
    MultiModalUUIDDict,
    NestedTensors,
)
from vllm.multimodal.parse import (
    AudioProcessorItems,
    MultiModalDataItems,
    MultiModalDataParser,
)
from vllm.multimodal.processing import (
    BaseMultiModalProcessor,
    BaseProcessingInfo,
    MultiModalProcessingInfo,
    PromptReplacement,
    PromptUpdate,
)
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.mistral import MistralTokenizer

from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
from .utils import init_vllm_registered_model, maybe_prefix

logger = init_logger(__name__)

ISO639_1_SUPPORTED_LANGS = {
    "ar": "Arabic",
    "nl": "Dutch",
    "en": "English",
    "fr": "French",
    "de": "German",
    "hi": "Hindi",
    "it": "Italian",
    "pt": "Portuguese",
    "es": "Spanish",
}

class VoxtralProcessorAdapter:
    """
    Provide a HF-compatible interface for
    :class:`mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
    """

    def __init__(self, tokenizer: MistralTokenizer) -> None:
        super().__init__()
        self.tokenizer = tokenizer

    @cached_property
    def _audio_processor(self) -> AudioEncoder:
        audio_encoder = self.tokenizer.instruct.audio_encoder
        assert isinstance(audio_encoder, AudioEncoder)
        return audio_encoder

    @cached_property
    def audio_token_id(self) -> int:
        return self._audio_processor.special_ids.audio

    @cached_property
    def begin_audio_token_id(self) -> int:
        return self._audio_processor.special_ids.begin_audio

    # @cached_property
    # def begin_transcript_token_id(self) -> int:
    #     return self._audio_processor.special_ids.begin_transcript

    # @cached_property
    # def end_transcript_token_id(self) -> int:
    #     return self._audio_processor.special_ids.end_transcript

    @cached_property
    def sampling_rate(self) -> int:
        return self._audio_processor.audio_config.sampling_rate

    @cached_property
    def frame_rate(self) -> float:
        return self._audio_processor.audio_config.frame_rate

    def get_num_audio_tokens(
        self,
        audio_length: int,
    ) -> int:
        pad_audio_length = self._audio_processor.next_multiple_of_chunk_frames(
            audio_length, self.sampling_


Summary: auto


**Exploring vLLM multimodal audio support**

Identified vLLM has multimodal classes potentially for audio transcription; next is inspecting its Python API and docs to understand how to pass audio data and configure transcription tasks. Planning to locate the vLLM package path and search for relevant keywords to uncover usage patterns or config options.


python3 - <<'PY'
import vllm, os
print(os.path.dirname(vllm.__file__))
PY


Output:
Exit code: 0
Wall time: 4.6 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/home/ubuntu/.local/lib/python3.10/site-packages/vllm


**Searching for transcription usage**


rg -n "transcription|audio" /home/ubuntu/.local/lib/python3.10/site-packages/vllm -S