Output:
Exit code: 0
Wall time: 0.1 seconds
Total output lines: 2196
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/inputs/preprocess.py:451:                # overridden by the audio features.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:25:from .audio import AudioResampler
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:28:    AudioItem,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:29:    HfAudioItem,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:219:class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:220:    def __init__(self, data: Sequence[HfAudioItem] | None) -> None:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:223:        super().__init__(data, "audio")
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:225:    def get_audio_length(self, item_idx: int) -> int:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:226:        audio = self.get(item_idx)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:227:        return len(audio)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:230:class AudioEmbeddingItems(EmbeddingItems):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:232:        super().__init__(data, "audio")
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:364:        target_sr (float, optional): Enables automatic resampling of audio
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:372:        audio_resample_method: Literal["librosa", "scipy"] = "librosa",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:377:        self.audio_resampler = AudioResampler(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:379:            method=audio_resample_method,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:402:    def _get_audio_with_sr(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:404:        audio: AudioItem,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:406:        if isinstance(audio, tuple):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:407:            return audio
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:408:        if isinstance(audio, list):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:409:            return np.array(audio), None
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:410:        if isinstance(audio, np.ndarray):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:411:            return audio, None
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:412:        if isinstance(audio, torch.Tensor):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:413:            return audio.numpy(), None
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:415:        assert_never(audio)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:432:    def _parse_audio_data(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:434:        data: ModalityData[AudioItem],
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:437:            return AudioProcessorItems(None)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:439:        # also check single audio item with sampling rate
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:446:            return AudioEmbeddingItems(data)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:448:        data_items: list[AudioItem]
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:461:        new_audios = list[np.ndarray]()
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:463:            audio, orig_sr = self._get_audio_with_sr(data_item)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:465:                new_audio = audio
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:467:                new_audio = self.audio_resampler.resample(audio, orig_sr=orig_sr)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:469:            new_audios.append(new_audio)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:471:        return AudioProcessorItems(new_audios)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/parse.py:548:            "audio": self._parse_audio_data,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/envs.py:69:    VLLM_AUDIO_FETCH_TIMEOUT: int = 10
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/envs.py:72:    VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/envs.py:751:    # Timeout for fetching audio when serving multimodal models
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/envs.py:753:    "VLLM_AUDIO_FETCH_TIMEOUT": lambda: int(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/envs.py:754:        os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "10")
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/envs.py:767:    # Maximum filesize in MB for a single audio file when processing
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/envs.py:770:    "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB": lambda: int(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/envs.py:771:        os.getenv("VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", "25")
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/envs.py:1678:        "VLLM_AUDIO_FETCH_TIMEOUT",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/envs.py:1681:        "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:24:from .audio import AudioEmbeddingMediaIO, AudioMediaIO
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:212:    def fetch_audio(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:214:        audio_url: str,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:217:        Load audio from a URL.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:219:        audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:222:            audio_url,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:223:            audio_io,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:224:            fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:227:    async def fetch_audio_async(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:229:        audio_url: str,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:232:        Asynchronously fetch audio from a URL.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:234:        audio_io = AudioMediaIO(**self.media_io_kwargs.get("audio", {}))
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:237:            audio_url,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:238:            audio_io,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:239:            fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:345:    def fetch_audio_embedding(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:350:        Load audio embedding from a URL.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:352:        audio_embedding_io = AudioEmbeddingMediaIO()
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:354:        return audio_embedding_io.load_base64("", data)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:357:def encode_audio_base64(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:358:    audio: np.ndarray,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:361:    """Encode audio as base64."""
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:362:    audio_io = AudioMediaIO()
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:363:    return audio_io.encode_base64((audio, sampling_rate))
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:453:def fetch_audio(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:454:    audio_url: str,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:455:    audio_io_kwargs: dict[str, Any] | None = None,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:459:        audio_url: URL of the audio file to fetch.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:460:        audio_io_kwargs: Additional kwargs passed to handle audio IO.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:466:    media_io_kwargs = None if not audio_io_kwargs else {"audio": audio_io_kwargs}
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/utils.py:471:    return media_connector.fetch_audio(audio_url)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:795:    - Audio: not yet supported.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:1784:            args.backend in ("openai-chat", "openai-audio")
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:1791:                "'openai-audio' backends."
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:2897:    Dataset class for processing a ASR dataset for transcription.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:2906:    | LibriSpeech    | Audiobook                              | Narrated                 | "LIUM/tedlium"              |
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:2907:    | GigaSpeech     | Audiobook, podcast, YouTube            | Narrated, spontaneous    | xs, s, m, l, xl, dev, test  |
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:2927:    TRANSCRIPTION_PREAMBLE = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:2928:    skip_long_audios: bool = True
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:2940:        prompt = ASRDataset.TRANSCRIPTION_PREAMBLE
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:2948:            audio = item["audio"]
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:2949:            y, sr = audio["array"], audio["sampling_rate"]
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:2952:            if self.skip_long_audios and duration_s > 30:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/datasets.py:2956:            mm_content = {"audio": (y, sr)}
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/processing.py:1100:        (text, image, audio...) with configurable options `kwargs`.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_engine.py:57:    TranscriptionRequest,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_engine.py:58:    TranscriptionResponse,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_engine.py:149:SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_engine.py:163:    | TranscriptionResponse
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_engine.py:843:            # up until '_', e.g., to match audio_url -> audio
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:29:def resample_audio_librosa(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:30:    audio: npt.NDArray[np.floating],
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:35:    return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:38:def resample_audio_scipy(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:39:    audio: npt.NDArray[np.floating],
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:48:        return scipy.signal.resample_poly(audio, 1, orig_sr // target_sr)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:50:        return scipy.signal.resample_poly(audio, target_sr // orig_sr, 1)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:51:    return audio
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:54:class AudioResampler:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:55:    """Resample audio data to a target sample rate."""
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:67:        audio: npt.NDArray[np.floating],
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:73:                "Audio resampling is not supported when `target_sr` is not provided"
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:76:            return resample_audio_librosa(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:77:                audio, orig_sr=orig_sr, target_sr=self.target_sr
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:80:            return resample_audio_scipy(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:81:                audio, orig_sr=orig_sr, target_sr=self.target_sr
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:90:class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:115:        audio, sr = media
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:118:            soundfile.write(buffer, audio, sr, format="WAV")
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/audio.py:124:class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:57:HfAudioItem: TypeAlias = Union[list[float], np.ndarray, "torch.Tensor"]
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:59:Represents a single audio
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:60:item, which can be passed to a HuggingFace `AudioProcessor`.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:86:AudioItem: TypeAlias = Union[HfAudioItem, tuple[np.ndarray, float], "torch.Tensor"]
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:88:Represents a single audio
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:89:item, which can be passed to a HuggingFace `AudioProcessor`.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:91:Alternatively, a tuple `(audio, sampling_rate)`, where the sampling rate
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:96:which are treated as audio embeddings;
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:120:    audio: ModalityData[AudioItem]
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:121:    """The input audio(s)."""
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/inputs.py:334:    """Based on the input, e.g., "image", "audio", "video"."""
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/profiling.py:13:    AudioDummyOptions,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/profiling.py:133:    def _get_dummy_audios(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/profiling.py:137:        num_audios: int,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/profiling.py:138:        overrides: AudioDummyOptions | None = None,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/profiling.py:140:        if num_audios == 0:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/profiling.py:145:                    "audio.length override (%d) exceeds model's "
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/profiling.py:151:        audio = np.zeros((length,))
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/multimodal/profiling.py:152:        return [audio] * num_audios
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/lib/endpoint_request_func.py:376:async def async_request_openai_audio(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/lib/endpoint_request_func.py:385:    _validate_api_url(api_url, "OpenAI Audio API", {"transcriptions", "translations"})
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/lib/endpoint_request_func.py:407:    # Send audio file
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/lib/endpoint_request_func.py:414:    mm_audio = request_func_input.multi_modal_content
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/lib/endpoint_request_func.py:415:    if not isinstance(mm_audio, dict) or "audio" not in mm_audio:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/lib/endpoint_request_func.py:416:        raise TypeError("multi_modal_content must be a dict containing 'audio'")
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/lib/endpoint_request_func.py:417:    with to_bytes(*mm_audio["audio"]) as f:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/lib/endpoint_request_func.py:419:        form.add_field("file", f, content_type="audio/wav")
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/benchmarks/lib/endpoint_request_func.py:761:    "openai-audio": async_request_openai_audio,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/mistral.py:68:    is_audio = bool(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/mistral.py:74:    assert not (is_vision and is_audio), "Vision and audio are mutually exclusive"
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/mistral.py:78:    if is_audio:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/mistral.py:79:        config_dict = _remap_mistral_audio_args(config_dict)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/mistral.py:183:def _remap_mistral_audio_args(config: dict) -> dict:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/mistral.py:193:        "audio_config": WhisperConfig(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/mistral.py:194:            num_mel_bins=encoder_args["audio_encoding_args"]["num_mel_bins"],
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/mistral.py:195:            window_size=encoder_args["audio_encoding_args"]["window_size"],
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/transformers_utils/configs/mistral.py:196:            sampling_rate=encode…71371 tokens truncated…py:437:        if not isinstance(audio_arrays, (torch.Tensor, list)):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:439:                f"Incorrect type of audio_arrays. Got type: {type(audio_arrays)}"
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:442:        if isinstance(audio_arrays, torch.Tensor):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:443:            audio_arrays = list(audio_arrays.unbind(0))
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:444:        return audio_arrays
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:457:        audio_config = tokenizer.instruct.audio_encoder.audio_config
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:458:        max_audio_clip_s = audio_config.chunk_length_s
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:459:        sample_rate = audio_config.sampling_rate
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:461:            max_audio_clip_s=max_audio_clip_s,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:468:    # for speech-to-text transcription
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:471:        audio: np.ndarray,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:480:        audio = Audio(audio, int(stt_config.sample_rate), format="wav")  # lossless
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:481:        req = TranscriptionRequest(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:483:            audio=RawAudio.from_audio(audio),
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:487:        tokenized = tokenizer.instruct.encode_transcription(req)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:488:        audio = (tokenized.audios[0].audio_array, stt_config.sample_rate)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:489:        prompts_dict = {"multi_modal_data": {"audio": audio}}
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:494:    def get_num_audio_tokens(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:496:        audio_duration_s: float,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:501:        Map from audio duration to number of audio tokens produced by the ASR
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:503:        This is used for estimating the amount of processing for this audio.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:507:        return adapter.get_num_audio_tokens(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:508:            int(audio_duration_s * stt_config.sample_rate)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:514:            (r"audio_language_projection\.(.*)", r"audio_language_adapter.\1"),
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:516:                r"audio_language_adapter\.0\.weight",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:517:                r"audio_language_adapter.w_in.weight",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:520:                r"audio_language_adapter\.2\.weight",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:521:                r"audio_language_adapter.w_out.weight",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:525:        audio_params = dict(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:528:                    "audio_language_adapter": self.audio_language_adapter,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:542:                        "mm_whisper_embeddings.audio_language_projection"
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:555:                if name in audio_params:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:556:                    param = audio_params[name]
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:626:                r"mm_whisper_embeddings\.audio_language_projection\.0",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:627:                r"audio_language_adapter.w_in",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:630:                r"mm_whisper_embeddings\.audio_language_projection\.2",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:631:                r"audio_language_adapter.w_out",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:664:class AudioLanguageAdapter(nn.Module):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:742:        audio_waveforms: torch.Tensor,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:744:        input_dtype = audio_waveforms.dtype
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:745:        window = torch.hann_window(self.config.window_size).to(audio_waveforms.device)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:747:            audio_waveforms,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:772:        audio_waveforms: list[torch.Tensor],
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:774:        assert isinstance(audio_waveforms, list)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:777:            self.compute_whisper_melspec(audio).to(self.dtype)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:778:            for audio in audio_waveforms
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:12:    Gemma3nAudioConfig,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:13:    Gemma3nAudioFeatureExtractor,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:57:from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsTranscription
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:69:TOKENS_PER_AUDIO = 188
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:85:class Gemma3nAudioInputs(TensorSchema):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:88:        - bn: Batch size * number of audios
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:93:    type: Literal["audio"] = "audio"
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:109:        return {"image": None, "audio": None}
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:114:        return {"image": TOKENS_PER_IMAGE, "audio": TOKENS_PER_AUDIO}
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:136:    def get_audio_repl(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:142:        Get the replacement text for audio tokens.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:144:        For Gemma3n, this should return the full_audio_sequence which includes
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:145:        BOA token, repeated audio tokens, and EOA token.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:150:        # Return the full audio sequence as defined by the processor
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:152:            processor.full_audio_sequence, processor.audio_token_id
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:159:        num_audios = mm_counts.get("audio", 0)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:163:        audio_token = processor.audio_token
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:165:        return image_token * num_images + audio_token * num_audios
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:174:        num_audios = mm_counts.get("audio", 0)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:176:        audio_feature_extractor: Gemma3nAudioFeatureExtractor = (
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:179:        audio_len = audio_feature_extractor.fft_length
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:185:        audio_overrides = mm_options.get("audio") if mm_options else None
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:194:            "audio": self._get_dummy_audios(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:195:                length=audio_len, num_audios=num_audios, overrides=audio_overrides
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:212:        # HF Transformers audio processor no longer accepts `audios` key.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:213:        # We pop `audios` and replace it with `audio` key to suppress
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:215:        if "audios" in mm_data:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:216:            mm_data["audio"] = mm_data.pop("audios")
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:225:            # Padding enables audio_tower to run in batched mode
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:249:            input_features_padded=MultiModalFieldConfig.batched("audio"),
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:250:            input_features_mask=MultiModalFieldConfig.batched("audio"),
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:284:        # Handle audio tokens
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:285:        if "audio" in mm_items:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:286:            audio_token = hf_processor.audio_token
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:288:            def get_replacement_audio(item_idx: int):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:289:                return self.info.get_audio_repl(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:295:                    modality="audio",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:296:                    target=audio_token,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:297:                    replacement=get_replacement_audio,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:390:        multimodal_config: Union[Gemma3nAudioConfig, Gemma3nVisionConfig],
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:464:    nn.Module, SupportsMultiModal, SupportsTranscription
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:483:            "model.embed_audio.": "embed_audio.",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:487:            "model.audio_tower.": "audio_tower.",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:505:        self.audio_tower = AutoModel.from_config(config=config.audio_config)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:509:        self.embed_audio = Gemma3nMultimodalEmbedder(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:510:            config.audio_config, config.text_config
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:542:    def _parse_and_validate_audio_input(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:544:    ) -> Optional[Gemma3nAudioInputs]:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:553:        return Gemma3nAudioInputs(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:573:                and "audio" not in mm_input_by_modality
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:575:                mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:606:    def _process_audio_input(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:608:        audio_input: Gemma3nAudioInputs,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:610:        assert self.audio_tower is not None
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:612:        input_features = audio_input["input_features_padded"].squeeze(1)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:613:        input_features_mask = audio_input["input_features_mask"].squeeze(1)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:614:        audio_outputs, audio_mask = self.audio_tower(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:617:        audio_features = self.embed_audio(inputs_embeds=audio_outputs)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:620:        # The Gemma3nProcessor expects all audio will be 30s in length and inserts 188 audio soft tokens into the
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:621:        # text to account for this. However, the audio preprocessing and encoder do not guarantee they will
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:623:        # depending on the length of the longest audio input in the batch. When we encounter this situation, we pad
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:624:        # the audio feature out to 188 soft tokens with the embedding of the last token in the embed_audio vocab.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:626:        audio_padding_toks = torch.tensor(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:627:            [[self.vocab_size - 1]], dtype=torch.long, device=audio_features.device
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:629:        audio_padding_embs = self.embed_audio(input_ids=audio_padding_toks)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:630:        audio_features = torch.where(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:631:            audio_mask.unsqueeze(-1), audio_padding_embs, audio_features
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:634:        audio_batch_size, audio_seq_len, audio_embed_dim = audio_features.shape
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:635:        extra_padding_tokens = self.config.audio_soft_tokens_per_image - audio_seq_len  # noqa: E501
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:636:        extra_padding_features = audio_padding_embs.expand(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:637:            audio_batch_size, extra_padding_tokens, audio_embed_dim
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:640:        audio_features = torch.cat((audio_features, extra_padding_features), dim=1)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:642:        return audio_features.unbind(0)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:661:            if modality == "audio":
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:662:                audio_embeddings = self._process_audio_input(multimodal_input)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:663:                multimodal_embeddings.extend(audio_embeddings)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:753:        elif modality == "audio":
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:754:            return "<audio_soft_token>"
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:761:        audio: np.ndarray,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:770:        Gemma3n supports "free-form" transcription.
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:771:        We fix its prompt here to standardize transcriptions/translations
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:774:        # Transcribe this audio [into <>] | for transcription
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:775:        # Translate this audio [from <> into <>] | for translation
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:778:        prompt += " this audio"
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:793:        prompt += ": <audio_soft_token><end_of_turn>\n<start_of_turn>model\n"
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:795:        audio = (audio, stt_config.sample_rate)
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:796:        prompts_dict = {"multi_modal_data": {"audio": audio}, "prompt": prompt}
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:806:            max_audio_clip_s=30,


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Assessing ASR integration via vLLM**

vLLM supports transcription for Gemma3n and Voxtral, enabling ASR backend addition using it. The user wants Huggingface model downloads with inference and full WER scoring without running transformer inference directly, favoring vLLM or similar. Integrating into the pipeline allows reuse of existing evaluation CLI, but a separate script might simplify running transcription evaluation with existing WER utilities.


**Planning minimal pipeline and CLI updates**

Adding CLI options for vLLM models will require small changes in asr, pipeline, and CLI modules without breaking defaults or patch instructions. I'll verify vLLM’s audio-related dependencies like `mistral_common`, `soundfile`, and `librosa` to ensure audio support is fully installed.


python3 - <<'PY'
import importlib.util
for mod in ["mistral_common", "librosa", "soundfile", "scipy"]:
    print(mod, importlib.util.find_spec(mod) is not None)
PY


Output:
Exit code: 0
Wall time: 0 seconds
Output:
mistral_common True
librosa True
soundfile True
scipy True


Summary: auto


**Inspecting vLLM transcription API**


python3 - <<'PY'
import inspect
from vllm import LLM
print([name for name, _ in inspect.getmembers(LLM, predicate=inspect.isfunction) if 'trans' in name.lower()][:20])
PY


Output:
Exit code: 124
Wall time: 10.1 seconds
Output:
command timed out after 10102 milliseconds
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-14 22:59:03.388753: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
[]


**Exploring vLLM transcription support**

I found that vLLM's LLM.generate can handle multimodal input with audio via SpeechToTextConfig, and models supporting transcription can use this with prompt plus multi_modal_data containing audio arrays. Next, I'll inspect vLLM's config and API for details on SpeechToTextConfig and transcription handling.


python3 - <<'PY'
import inspect
import vllm.config
print(inspect.getsource(vllm.config.SpeechToTextConfig)[:4000])
PY


Output:
Exit code: 0
Wall time: 10.2 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-14 22:59:20.491654: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
@config
@dataclass
class SpeechToTextConfig:
    """Configuration for speech-to-text models."""

    sample_rate: float = 16_000
    """Sample rate (Hz) to resample input audio to. Most speech models expect
    16kHz audio input. The input audio will be automatically resampled to this
    rate before processing."""

    max_audio_clip_s: int = 30
    """Maximum duration in seconds for a single audio clip without chunking.
    Audio longer than this will be split into smaller chunks if
    `allow_audio_chunking` evaluates to True, otherwise it will be rejected."""

    overlap_chunk_second: int = 1
    """Overlap duration in seconds between consecutive audio chunks when
    splitting long audio. This helps maintain context across chunk boundaries
    and improves transcription quality at split points."""

    min_energy_split_window_size: int | None = 1600
    """Window size in samples for finding low-energy (quiet) regions to split
    audio chunks. The algorithm looks for the quietest moment within this
    window to minimize cutting through speech. Default 1600 samples ≈ 100ms
    at 16kHz. If None, no chunking will be done."""

    @property
    def allow_audio_chunking(self) -> bool:
        return self.min_energy_split_window_size is not None


**Confirming speech-to-text config usage**

I'm checking if `vllm.LLM` accepts `speech_to_text_config` by searching for `SpeechToTextConfig` in the vllm codebase to clarify how the config integrates with the LLM.


rg -n "SpeechToTextConfig" /home/ubuntu/.local/lib/python3.10/site-packages/vllm -S


Output:
Exit code: 0
Wall time: 0 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/config/__init__.py:30:from vllm.config.speech_to_text import SpeechToTextConfig
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/config/__init__.py:90:    "SpeechToTextConfig",
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/config/speech_to_text.py:12:class SpeechToTextConfig:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/whisper.py:22:from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/whisper.py:821:        stt_config: SpeechToTextConfig,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/whisper.py:857:    ) -> SpeechToTextConfig:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/whisper.py:860:        return SpeechToTextConfig(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/whisper.py:869:        stt_config: SpeechToTextConfig,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/interfaces.py:22:from vllm.config import ModelConfig, SpeechToTextConfig
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/interfaces.py:892:        stt_config: SpeechToTextConfig,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/interfaces.py:937:    ) -> SpeechToTextConfig:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/interfaces.py:945:        stt_config: SpeechToTextConfig,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:21:from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:762:        stt_config: SpeechToTextConfig,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:802:    ) -> SpeechToTextConfig:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py:803:        return SpeechToTextConfig(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/granite_speech.py:37:from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/granite_speech.py:844:        stt_config: SpeechToTextConfig,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/granite_speech.py:884:        stt_config: SpeechToTextConfig,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/granite_speech.py:907:    ) -> SpeechToTextConfig:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/granite_speech.py:912:        return SpeechToTextConfig()
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:23:from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:455:    ) -> SpeechToTextConfig:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:460:        return SpeechToTextConfig(
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:473:        stt_config: SpeechToTextConfig,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:497:        stt_config: SpeechToTextConfig,


**Inspecting transcription interface in vLLM**


python3 - <<'PY'
import inspect
from vllm.model_executor.models import interfaces
src = inspect.getsource(interfaces)
# Print snippet around SupportsTranscription
start = src.find('class SupportsTranscription')
print(src[start:start+2000])
PY


Output:
Exit code: 0
Wall time: 10.6 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-14 22:59:37.415157: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
class SupportsTranscription(Protocol):
    """The interface required for all models that support transcription."""

    # Mapping from ISO639_1 language codes: language names
    supported_languages: ClassVar[Mapping[str, str]]

    supports_transcription: ClassVar[Literal[True]] = True

    supports_transcription_only: ClassVar[bool] = False
    """
    Transcription models can opt out of text generation by setting this to
    `True`.
    """
    supports_segment_timestamp: ClassVar[bool] = False
    """
    Enables the segment timestamp option for supported models by setting this to `True`.
    """

    def __init_subclass__(cls, **kwargs):
        super().__init_subclass__(**kwargs)
        # language codes in supported_languages
        # that don't exist in the full language map
        invalid = set(cls.supported_languages) - set(LANGUAGES.keys())
        if invalid:
            raise ValueError(
                f"{cls.__name__}.supported_languages contains invalid "
                f"language codes: {sorted(invalid)}\n. "
                f"Valid choices are: {sorted(LANGUAGES.keys())}"
            )

    @classmethod
    def get_generation_prompt(
        cls,
        audio: np.ndarray,
        stt_config: SpeechToTextConfig,
        model_config: ModelConfig,
        language: str | None,
        task_type: Literal["transcribe", "translate"],
        request_prompt: str,
        to_language: str | None,
    ) -> PromptType:
        """Get the prompt for the ASR model.
        The model has control over the construction, as long as it
        returns a valid PromptType."""
        ...

    @classmethod
    def get_other_languages(cls) -> Mapping[str, str]:
        # other possible language codes from the whisper map
        return {k: v for k, v in LANGUAGES.items() if k not in cls.supported_languages}

    @classmethod
    def validate_language(cls, language: str | None) -> str | None:
        """
        Ensure the language specified in the transcription re


python3 - <<'PY'
import inspect
from vllm.model_executor.models import interfaces
src = inspect.getsource(interfaces)
# show part with get_generation_prompt usage
idx = src.find('get_generation_prompt')
print(src[idx:idx+1800])
PY


Output:
Exit code: 0
Wall time: 10.2 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
2026-01-14 22:59:53.825882: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
/home/ubuntu/.local/lib/python3.10/site-packages/google/api_core/_python_version_support.py:266: FutureWarning: You are using a Python version (3.10.12) which Google will stop supporting in new releases of google.api_core once it reaches its end of life (2026-10-04). Please upgrade to the latest Python version, or at least Python 3.11, to continue receiving updates for google.api_core past that date.
  warnings.warn(message, FutureWarning)
get_generation_prompt(
        cls,
        audio: np.ndarray,
        stt_config: SpeechToTextConfig,
        model_config: ModelConfig,
        language: str | None,
        task_type: Literal["transcribe", "translate"],
        request_prompt: str,
        to_language: str | None,
    ) -> PromptType:
        """Get the prompt for the ASR model.
        The model has control over the construction, as long as it
        returns a valid PromptType."""
        ...

    @classmethod
    def get_other_languages(cls) -> Mapping[str, str]:
        # other possible language codes from the whisper map
        return {k: v for k, v in LANGUAGES.items() if k not in cls.supported_languages}

    @classmethod
    def validate_language(cls, language: str | None) -> str | None:
        """
        Ensure the language specified in the transcription request
        is a valid ISO 639-1 language code. If the request language is
        valid, but not natively supported by the model, trigger a
        warning (but not an exception).
        """
        if language is None or language in cls.supported_languages:
            return language
        elif language in cls.get_other_languages():
            logger.warning(
                "Language %r is not natively supported by %s; "
                "results may be less accurate. Supported languages: %r",
                language,
                cls.__name__,
                list(cls.supported_languages.keys()),
            )
            return language
        else:
            raise ValueError(
                f"Unsupported language: {language!r}.  Must be one of "
                f"{list(cls.supported_languages.keys())}."
            )

    @classmethod
    def get_speech_to_text_config(
        cls, model_config: ModelConfig, task_typ


**Investigating transcription call flow**

I'm exploring how to create and pass prompts for transcription, focusing on `get_generation_prompt` and whether `vllm` offers input helpers or an API method like `LLM.transcribe`. I'll examine `vllm/entrypoints/openai/serving_engine.py` and inspect input data types in `vllm/inputs/data.py` to understand how to supply audio for generation calls.


python3 - <<'PY'
import inspect
from vllm.inputs import data
print(inspect.getsource(data.PromptType)[:600])
PY


Output:
Exit code: 1
Wall time: 4.6 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
Traceback (most recent call last):
  File "<stdin>", line 3, in <module>
  File "/usr/lib/python3.10/inspect.py", line 1139, in getsource
    lines, lnum = getsourcelines(object)
  File "/usr/lib/python3.10/inspect.py", line 1121, in getsourcelines
    lines, lnum = findsource(object)
  File "/usr/lib/python3.10/inspect.py", line 940, in findsource
    file = getsourcefile(object)
  File "/usr/lib/python3.10/inspect.py", line 817, in getsourcefile
    filename = getfile(object)
  File "/home/ubuntu/.local/lib/python3.10/site-packages/torch/package/package_importer.py", line 730, in _patched_getfile
    return _orig_getfile(object)
  File "/usr/lib/python3.10/inspect.py", line 797, in getfile
    raise TypeError('module, class, method, function, traceback, frame, or '
TypeError: module, class, method, function, traceback, frame, or code object was expected, got UnionType


**Switching to file inspection**

Since the typing union can't be inspected directly, I'll open the file to analyze the PromptType definition.


python3 - <<'PY'
import vllm.inputs.data as data
import inspect, os
print(os.path.abspath(inspect.getsourcefile(data)))
PY


Output:
Exit code: 0
Wall time: 4.6 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/inputs/data.py


sed -n '1,200p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/inputs/data.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, cast

import torch
from typing_extensions import NotRequired, TypedDict, TypeIs, TypeVar

if TYPE_CHECKING:
    from vllm.multimodal.inputs import (
        MultiModalDataDict,
        MultiModalInputs,
        MultiModalUUIDDict,
    )
else:
    MultiModalDataDict = object
    MultiModalInputs = object
    MultiModalUUIDDict = object

class TextPrompt(TypedDict):
    """Schema for a text prompt."""

    prompt: str
    """The input text to be tokenized before passing to the model."""

    multi_modal_data: NotRequired[MultiModalDataDict | None]
    """
    Optional multi-modal data to pass to the model,
    if the model supports it.
    """

    mm_processor_kwargs: NotRequired[dict[str, Any] | None]
    """
    Optional multi-modal processor kwargs to be forwarded to the
    multimodal input mapper & processor. Note that if multiple modalities
    have registered mappers etc for the model being considered, we attempt
    to pass the mm_processor_kwargs to each of them.
    """

    multi_modal_uuids: NotRequired[MultiModalUUIDDict]
    """
    Optional user-specified UUIDs for multimodal items, mapped by modality.
    Lists must match the number of items per modality and may contain `None`.
    For `None` entries, the hasher will compute IDs automatically; non-None
    entries override the default hashes for caching, and MUST be unique per
    multimodal item.
    """

    cache_salt: NotRequired[str]
    """
    Optional cache salt to be used for prefix caching.
    """

class TokensPrompt(TypedDict):
    """Schema for a tokenized prompt."""

    prompt_token_ids: list[int]
    """A list of token IDs to pass to the model."""

    prompt: NotRequired[str]
    """The prompt text corresponding to the token IDs, if available."""

    token_type_ids: NotRequired[list[int]]
    """A list of token type IDs to pass to the cross encoder model."""

    multi_modal_data: NotRequired[MultiModalDataDict | None]
    """
    Optional multi-modal data to pass to the model,
    if the model supports it.
    """

    mm_processor_kwargs: NotRequired[dict[str, Any] | None]
    """
    Optional multi-modal processor kwargs to be forwarded to the
    multimodal input mapper & processor. Note that if multiple modalities
    have registered mappers etc for the model being considered, we attempt
    to pass the mm_processor_kwargs to each of them.
    """

    multi_modal_uuids: NotRequired[MultiModalUUIDDict]
    """
    Optional user-specified UUIDs for multimodal items, mapped by modality.
    Lists must match the number of items per modality and may contain `None`.
    For `None` entries, the hasher will compute IDs automatically; non-None
    entries override the default hashes for caching.
    """

    cache_salt: NotRequired[str]
    """
    Optional cache salt to be used for prefix caching.
    """

class EmbedsPrompt(TypedDict):
    """Schema for a prompt provided via token embeddings."""

    prompt_embeds: torch.Tensor
    """The embeddings of the prompt."""

    cache_salt: NotRequired[str]
    """
    Optional cache salt to be used for prefix caching.
    """

class DataPrompt(TypedDict):
    """Represents generic inputs handled by IO processor plugins."""

    data: Any
    """The input data"""

    data_format: str
    """The input data format"""

SingletonPrompt: TypeAlias = str | TextPrompt | TokensPrompt | EmbedsPrompt
"""
Set of possible schemas for a single prompt:

- A text prompt ([`str`][] or [`TextPrompt`][vllm.inputs.data.TextPrompt])
- A tokenized prompt ([`TokensPrompt`][vllm.inputs.data.TokensPrompt])
- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.data.EmbedsPrompt])

Note that "singleton" is as opposed to a data structure
which encapsulates multiple prompts, i.e. of the sort
which may be utilized for encoder/decoder models when
the user desires to express both the encoder & decoder
prompts explicitly, i.e.
[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]

A prompt of type [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] may be
employed as (1) input to a decoder-only model, (2) input to
the encoder of an encoder/decoder model, in the scenario
where the decoder-prompt is not specified explicitly, or
(3) as a member of a larger data structure encapsulating
more than one prompt, i.e.
[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]
"""

def is_tokens_prompt(prompt: SingletonPrompt) -> TypeIs[TokensPrompt]:
    return (
        isinstance(prompt, dict)
        and "prompt_token_ids" in prompt
        and "prompt_embeds" not in prompt
    )

def is_embeds_prompt(prompt: SingletonPrompt) -> TypeIs[EmbedsPrompt]:
    return (
        isinstance(prompt, dict)
        and "prompt_token_ids" not in prompt
        and "prompt_embeds" in prompt
    )

_T1_co = TypeVar(
    "_T1_co", bound=SingletonPrompt, default=SingletonPrompt, covariant=True
)
_T2_co = TypeVar(
    "_T2_co", bound=SingletonPrompt, default=SingletonPrompt, covariant=True
)

# TODO: Make fields ReadOnly once mypy supports it
class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
    """
    Represents an encoder/decoder model input prompt,
    comprising an explicit encoder prompt and a decoder prompt.

    The encoder and decoder prompts, respectively, may be formatted
    according to any of the
    [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] schemas,
    and are not required to have the same schema.

    Only the encoder prompt may have multi-modal data. mm_processor_kwargs
    should be at the top-level, and should not be set in the encoder/decoder
    prompts, since they are agnostic to the encoder/decoder.

    Note that an
    [`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]
    may not be used as an input to a decoder-only model,
    and that the `encoder_prompt` and `decoder_prompt`
    fields of this data structure themselves must be
    [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] instances.
    """

    encoder_prompt: _T1_co

    decoder_prompt: _T2_co | None

    mm_processor_kwargs: NotRequired[dict[str, Any]]

PromptType: TypeAlias = SingletonPrompt | ExplicitEncoderDecoderPrompt
"""
Set of possible schemas for an LLM input, including
both decoder-only and encoder/decoder input types:


**Assessing transcription prompt strategies**

I'm evaluating several approaches for generating transcription prompts, including using model-specific `get_generation_prompt` methods, mimicking vLLM's serving engine logic, or leveraging vLLM's OpenAI-style audio transcription API; I'll explore whether in-process calls to vLLM's transcription endpoints can simplify this without running a full server.


**Inspecting transcription in serving engine**


sed -n '740,920p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_engine.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
        except Exception as e:
            return self.create_error_response(str(e))

    def create_error_response(
        self,
        message: str,
        err_type: str = "BadRequestError",
        status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
    ) -> ErrorResponse:
        if self.log_error_stack:
            exc_type, _, _ = sys.exc_info()
            if exc_type is not None:
                traceback.print_exc()
            else:
                traceback.print_stack()
        return ErrorResponse(
            error=ErrorInfo(message=message, type=err_type, code=status_code.value)
        )

    def create_streaming_error_response(
        self,
        message: str,
        err_type: str = "BadRequestError",
        status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
    ) -> str:
        json_str = json.dumps(
            self.create_error_response(
                message=message, err_type=err_type, status_code=status_code
            ).model_dump()
        )
        return json_str

    def _raise_if_error(self, finish_reason: str | None, request_id: str) -> None:
        """Raise GenerationError if finish_reason indicates an error."""
        if finish_reason == "error":
            logger.error(
                "Request %s failed with an internal error during generation",
                request_id,
            )
            raise GenerationError("Internal server error")

    def _convert_generation_error_to_response(
        self, e: GenerationError
    ) -> ErrorResponse:
        """Convert GenerationError to ErrorResponse."""
        return self.create_error_response(
            str(e),
            err_type="InternalServerError",
            status_code=e.status_code,
        )

    def _convert_generation_error_to_streaming_response(
        self, e: GenerationError
    ) -> str:
        """Convert GenerationError to streaming error response."""
        return self.create_streaming_error_response(
            str(e),
            err_type="InternalServerError",
            status_code=e.status_code,
        )

    async def _check_model(
        self,
        request: AnyRequest,
    ) -> ErrorResponse | None:
        error_response = None

        if self._is_model_supported(request.model):
            return None
        if request.model in self.models.lora_requests:
            return None
        if (
            envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
            and request.model
            and (load_result := await self.models.resolve_lora(request.model))
        ):
            if isinstance(load_result, LoRARequest):
                return None
            if (
                isinstance(load_result, ErrorResponse)
                and load_result.error.code == HTTPStatus.BAD_REQUEST.value
            ):
                error_response = load_result

        return error_response or self.create_error_response(
            message=f"The model `{request.model}` does not exist.",
            err_type="NotFoundError",
            status_code=HTTPStatus.NOT_FOUND,
        )

    def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
        """Determine if there are any active default multimodal loras."""
        # TODO: Currently this is only enabled for chat completions
        # to be better aligned with only being enabled for .generate
        # when run offline. It would be nice to support additional
        # tasks types in the future.
        message_types = self._get_message_types(request)
        default_mm_loras = set()

        for lora in self.models.lora_requests.values():
            # Best effort match for default multimodal lora adapters;
            # There is probably a better way to do this, but currently
            # this matches against the set of 'types' in any content lists
            # up until '_', e.g., to match audio_url -> audio
            if lora.lora_name in message_types:
                default_mm_loras.add(lora)

        # Currently only support default modality specific loras if
        # we have exactly one lora matched on the request.
        if len(default_mm_loras) == 1:
            return default_mm_loras.pop()
        return None

    def _maybe_get_adapters(
        self,
        request: AnyRequest,
        supports_default_mm_loras: bool = False,
    ) -> LoRARequest | None:
        if request.model in self.models.lora_requests:
            return self.models.lora_requests[request.model]

        # Currently only support default modality specific loras
        # if we have exactly one lora matched on the request.
        if supports_default_mm_loras:
            default_mm_lora = self._get_active_default_mm_loras(request)
            if default_mm_lora is not None:
                return default_mm_lora

        if self._is_model_supported(request.model):
            return None

        # if _check_model has been called earlier, this will be unreachable
        raise ValueError(f"The model `{request.model}` does not exist.")

    def _get_message_types(self, request: AnyRequest) -> set[str]:
        """Retrieve the set of types from message content dicts up
        until `_`; we use this to match potential multimodal data
        with default per modality loras.
        """
        message_types: set[str] = set()

        if not hasattr(request, "messages"):
            return message_types

        messages = request.messages
        if messages is None or isinstance(messages, (str, bytes)):
            return message_types

        for message in messages:
            if (
                isinstance(message, dict)
                and "content" in message
                and isinstance(message["content"], list)
            ):
                for content_dict in message["content"]:
                    if "type" in content_dict:
                        message_types.add(content_dict["type"].split("_")[0])
        return message_types

    async def _normalize_prompt_text_to_input(
        self,
        request: AnyRequest,
        prompt: str,
        tokenizer: TokenizerLike,
        add_special_tokens: bool,
    ) -> TokensPrompt:
        async_tokenizer = self._get_async_tokenizer(tokenizer)

        if (
            self.model_config.encoder_config is not None
            and self.model_config.encoder_config.get("do_lower_case", False)
        ):
            prompt = prompt.lower()

        truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)

        if truncate_prompt_tokens is None:
            encoded = await async_tokenizer(
                prompt, add_special_tokens=add_special_tokens
            )
        elif truncate_prompt_tokens < 0:


Summary: auto


**Searching transcription-related terms**

I'm searching the large file for references to transcription, focusing on `TranscriptionRequest` or `SpeechToTextRequest` to locate relevant sections.


rg -n "transcription" /home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_engine.py -n


Output:
Exit code: 1
Wall time: 0 seconds
Output:


**Searching for transcription request usage**


Output:
Exit code: 0
Wall time: 0 seconds
Output:
57:    TranscriptionRequest,
149:SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
153:    | SpeechToTextRequest


**Exploring vLLM audio transcription options**

I'm considering using vLLM's OpenAI API server or direct protocol calls for transcription but note complexity; next, I'll check vLLM's SpeechToText support and how it uses `SupportsTranscription` and `speech_to_text_config` to handle audio preprocessing.


rg -n "SpeechToText" /home/ubuntu/.local/lib/python3.10/site-packages/vllm/inputs -S


Output:
Exit code: 1
Wall time: 0 seconds
Output:


**Investigating audio-related code in preprocess.py**


rg -n "speech|audio" /home/ubuntu/.local/lib/python3.10/site-packages/vllm/inputs/preprocess.py -n


Output:
Exit code: 0
Wall time: 0 seconds
Output:
451:                # overridden by the audio features.


rg -n "TranscriptionRequest" /home/ubuntu/.local/lib/python3.10/site-packages/vllm -S


Output:
Exit code: 0
Wall time: 0 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_transcription.py:12:    TranscriptionRequest,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_transcription.py:55:        self, audio_data: bytes, request: TranscriptionRequest, raw_request: Request
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_transcription.py:81:        request: TranscriptionRequest,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_engine.py:57:    TranscriptionRequest,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_engine.py:149:SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py:58:    TranscriptionRequest,
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py:576:    raw_request: Request, request: Annotated[TranscriptionRequest, Form()]
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/protocol.py:1949:class TranscriptionRequest(OpenAIBaseModel):
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/whisper.py:811:                "in the TranscriptionRequest."
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:18:from mistral_common.protocol.transcription.request import TranscriptionRequest
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py:481:        req = TranscriptionRequest(


**Inspecting existing transcription utilities**

I'll check the serving_transcription module in vLLM to see if its in-process functions can simplify integration.


sed -n '1,240p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_transcription.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import AsyncGenerator

from fastapi import Request

from vllm.engine.protocol import EngineClient
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.protocol import (
    ErrorResponse,
    RequestResponseMetadata,
    TranscriptionRequest,
    TranscriptionResponse,
    TranscriptionResponseStreamChoice,
    TranscriptionResponseVerbose,
    TranscriptionStreamResponse,
    TranslationRequest,
    TranslationResponse,
    TranslationResponseStreamChoice,
    TranslationResponseVerbose,
    TranslationStreamResponse,
)
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.entrypoints.openai.speech_to_text import OpenAISpeechToText
from vllm.logger import init_logger
from vllm.outputs import RequestOutput

logger = init_logger(__name__)

class OpenAIServingTranscription(OpenAISpeechToText):
    """Handles transcription requests."""

    def __init__(
        self,
        engine_client: EngineClient,
        models: OpenAIServingModels,
        *,
        request_logger: RequestLogger | None,
        return_tokens_as_token_ids: bool = False,
        log_error_stack: bool = False,
        enable_force_include_usage: bool = False,
    ):
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            task_type="transcribe",
            log_error_stack=log_error_stack,
            enable_force_include_usage=enable_force_include_usage,
        )

    async def create_transcription(
        self, audio_data: bytes, request: TranscriptionRequest, raw_request: Request
    ) -> (
        TranscriptionResponse
        | TranscriptionResponseVerbose
        | AsyncGenerator[str, None]
        | ErrorResponse
    ):
        """Transcription API similar to OpenAI's API.

        See https://platform.openai.com/docs/api-reference/audio/createTranscription
        for the API specification. This API mimics the OpenAI transcription API.
        """
        return await self._create_speech_to_text(
            audio_data=audio_data,
            request=request,
            raw_request=raw_request,
            response_class=(
                TranscriptionResponseVerbose
                if request.response_format == "verbose_json"
                else TranscriptionResponse
            ),
            stream_generator_method=self.transcription_stream_generator,
        )

    async def transcription_stream_generator(
        self,
        request: TranscriptionRequest,
        result_generator: list[AsyncGenerator[RequestOutput, None]],
        request_id: str,
        request_metadata: RequestResponseMetadata,
        audio_duration_s: float,
    ) -> AsyncGenerator[str, None]:
        generator = self._speech_to_text_stream_generator(
            request=request,
            list_result_generator=result_generator,
            request_id=request_id,
            request_metadata=request_metadata,
            audio_duration_s=audio_duration_s,
            chunk_object_type="transcription.chunk",
            response_stream_choice_class=TranscriptionResponseStreamChoice,
            stream_response_class=TranscriptionStreamResponse,
        )
        async for chunk in generator:
            yield chunk

class OpenAIServingTranslation(OpenAISpeechToText):
    """Handles translation requests."""

    def __init__(
        self,
        engine_client: EngineClient,
        models: OpenAIServingModels,
        *,
        request_logger: RequestLogger | None,
        return_tokens_as_token_ids: bool = False,
        log_error_stack: bool = False,
        enable_force_include_usage: bool = False,
    ):
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            task_type="translate",
            log_error_stack=log_error_stack,
            enable_force_include_usage=enable_force_include_usage,
        )

    async def create_translation(
        self, audio_data: bytes, request: TranslationRequest, raw_request: Request
    ) -> (
        TranslationResponse
        | TranslationResponseVerbose
        | AsyncGenerator[str, None]
        | ErrorResponse
    ):
        """Translation API similar to OpenAI's API.

        See https://platform.openai.com/docs/api-reference/audio/createTranslation
        for the API specification. This API mimics the OpenAI translation API.
        """
        return await self._create_speech_to_text(
            audio_data=audio_data,
            request=request,
            raw_request=raw_request,
            response_class=(
                TranslationResponseVerbose
                if request.response_format == "verbose_json"
                else TranslationResponse
            ),
            stream_generator_method=self.translation_stream_generator,
        )

    async def translation_stream_generator(
        self,
        request: TranslationRequest,
        result_generator: list[AsyncGenerator[RequestOutput, None]],
        request_id: str,
        request_metadata: RequestResponseMetadata,
        audio_duration_s: float,
    ) -> AsyncGenerator[str, None]:
        generator = self._speech_to_text_stream_generator(
            request=request,
            list_result_generator=result_generator,
            request_id=request_id,
            request_metadata=request_metadata,
            audio_duration_s=audio_duration_s,
            chunk_object_type="translation.chunk",
            response_stream_choice_class=TranslationResponseStreamChoice,
            stream_response_class=TranslationStreamResponse,
        )
        async for chunk in generator:
            yield chunk


**Inspecting speech-to-text entrypoint**

I'm checking the `OpenAISpeechToText` implementation in `vllm/entrypoints/openai/speech_to_text.py` to understand how the internal OpenAI API is leveraged for speech-to-text.


sed -n '1,240p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/speech_to_text.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import io
import math
import time
from collections.abc import AsyncGenerator, Callable
from functools import cached_property
from typing import Literal, TypeAlias, TypeVar, cast

import numpy as np
from fastapi import Request
from transformers import PreTrainedTokenizerBase

import vllm.envs as envs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.protocol import (
    DeltaMessage,
    ErrorResponse,
    RequestResponseMetadata,
    TranscriptionResponse,
    TranscriptionResponseStreamChoice,
    TranscriptionResponseVerbose,
    TranscriptionSegment,
    TranscriptionStreamResponse,
    TranslationResponse,
    TranslationResponseStreamChoice,
    TranslationResponseVerbose,
    TranslationSegment,
    TranslationStreamResponse,
    UsageInfo,
)
from vllm.entrypoints.openai.serving_engine import OpenAIServing, SpeechToTextRequest
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.inputs.data import PromptType
from vllm.logger import init_logger
from vllm.model_executor.models import SupportsTranscription
from vllm.outputs import RequestOutput
from vllm.tokenizers import get_tokenizer
from vllm.utils.import_utils import PlaceholderModule

try:
    import librosa
except ImportError:
    librosa = PlaceholderModule("librosa")  # type: ignore[assignment]

SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
SpeechToTextResponseVerbose: TypeAlias = (
    TranscriptionResponseVerbose | TranslationResponseVerbose
)
SpeechToTextSegment: TypeAlias = TranscriptionSegment | TranslationSegment
T = TypeVar("T", bound=SpeechToTextResponse)
V = TypeVar("V", bound=SpeechToTextResponseVerbose)
S = TypeVar("S", bound=SpeechToTextSegment)

ResponseType: TypeAlias = (
    TranscriptionResponse
    | TranslationResponse
    | TranscriptionResponseVerbose
    | TranslationResponseVerbose
)

logger = init_logger(__name__)

class OpenAISpeechToText(OpenAIServing):
    """Base class for speech-to-text operations like transcription and
    translation."""

    def __init__(
        self,
        engine_client: EngineClient,
        models: OpenAIServingModels,
        *,
        request_logger: RequestLogger | None,
        return_tokens_as_token_ids: bool = False,
        task_type: Literal["transcribe", "translate"] = "transcribe",
        log_error_stack: bool = False,
        enable_force_include_usage: bool = False,
    ):
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )

        self.default_sampling_params = self.model_config.get_diff_sampling_param()
        self.task_type = task_type

        self.asr_config = self.model_cls.get_speech_to_text_config(
            self.model_config, task_type
        )

        self.enable_force_include_usage = enable_force_include_usage

        self.max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
        if self.model_cls.supports_segment_timestamp:
            self.tokenizer = cast(
                PreTrainedTokenizerBase,
                get_tokenizer(
                    tokenizer_name=self.model_config.tokenizer,
                    tokenizer_mode=self.model_config.tokenizer_mode,
                ),
            )

        if self.default_sampling_params:
            logger.info(
                "Overwriting default completion sampling param with: %s",
                self.default_sampling_params,
            )

    @cached_property
    def model_cls(self) -> type[SupportsTranscription]:
        from vllm.model_executor.model_loader import get_model_cls

        model_cls = get_model_cls(self.model_config)
        return cast(type[SupportsTranscription], model_cls)

    async def _preprocess_speech_to_text(
        self,
        request: SpeechToTextRequest,
        audio_data: bytes,
    ) -> tuple[list[PromptType], float]:
        # Validate request
        language = self.model_cls.validate_language(request.language)
        # Skip to_language validation to avoid extra logging for Whisper.
        to_language = (
            self.model_cls.validate_language(request.to_language)
            if request.to_language
            else None
        )

        if len(audio_data) / 1024**2 > self.max_audio_filesize_mb:
            raise ValueError("Maximum file size exceeded.")

        with io.BytesIO(audio_data) as bytes_:
            # NOTE resample to model SR here for efficiency. This is also a
            # pre-requisite for chunking, as it assumes Whisper SR.
            y, sr = librosa.load(bytes_, sr=self.asr_config.sample_rate)

        duration = librosa.get_duration(y=y, sr=sr)
        do_split_audio = (
            self.asr_config.allow_audio_chunking
            and duration > self.asr_config.max_audio_clip_s
        )
        chunks = [y] if not do_split_audio else self._split_audio(y, int(sr))
        prompts = []
        for chunk in chunks:
            # The model has control over the construction, as long as it
            # returns a valid PromptType.
            prompt = self.model_cls.get_generation_prompt(
                audio=chunk,
                stt_config=self.asr_config,
                model_config=self.model_config,
                language=language,
                task_type=self.task_type,
                request_prompt=request.prompt,
                to_language=to_language,
            )
            if request.response_format == "verbose_json":
                if not isinstance(prompt, dict):
                    raise ValueError(f"Expected prompt to be a dict,got {type(prompt)}")
                prompt_dict = cast(dict, prompt)
                decoder_prompt = prompt.get("decoder_prompt")
                if not isinstance(decoder_prompt, str):
                    raise ValueError(
                        f"Expected decoder_prompt to bestr, got {type(decoder_prompt)}"
                    )
                prompt_dict["decoder_prompt"] = decoder_prompt.replace(
                    "<|notimestamps|>", "<|0.00|>"
                )
            prompts.append(prompt)
        return prompts, duration

    def _get_verbose_segments(
        self,
        tokens: tuple,
        request: SpeechToTextRequest,
        segment_class: type[SpeechToTextSegment],
        start_time: float = 0,
    ) -> list[SpeechToTextSegment]:
        """
        Convert tokens to verbose segments.

        This method expects the model to produce
        timestamps as tokens (similar to Whisper).
        If the tokens do not include timestamp information,
        the segments may not be generated correctly.

        Note: Fields like avg_logprob, compression_ratio,
        and no_speech_prob are not supported
        in this implementation and will be None. See docs for details.
        """
        BASE_OFFSET = 0.02
        init_token = self.tokenizer.encode("<|0.00|>", add_special_tokens=False)[0]
        if tokens[-1] == self.tokenizer.eos_token_id:
            tokens = tokens[:-1]

        tokens_with_start = (init_token,) + tokens
        segments: list[SpeechToTextSegment] = []
        last_timestamp_start = 0

        if tokens_with_start[-2] < init_token and tokens_with_start[-1] >= init_token:
            tokens_with_start = tokens_with_start + (tokens_with_start[-1],)
        for idx, token in enumerate(tokens_with_start):
            # Timestamp tokens (e.g., <|0.00|>) are assumed to be sorted.
            # If the ordering is violated, this slicing may produce incorrect results.
            if (
                token >= init_token
                and idx != 0
                and tokens_with_start[idx - 1] >= init_token
            ):
                sliced_timestamp_tokens = tokens_with_start[last_timestamp_start:idx]
                start_timestamp = sliced_timestamp_tokens[0] - init_token
                end_timestamp = sliced_timestamp_tokens[-1] - init_token

                casting_segment = cast(
                    SpeechToTextSegment,
                    segment_class(
                        id=len(segments),
                        seek=start_time,
                        start=start_time + BASE_OFFSET * start_timestamp,
                        end=start_time + BASE_OFFSET * end_timestamp,
                        temperature=request.temperature,
                        text=self.tokenizer.decode(sliced_timestamp_tokens[1:-1]),
                        tokens=sliced_timestamp_tokens[1:-1],
                    ),
                )
                segments.append(casting_segment)
                last_timestamp_start = idx
        return segments

    async def _create_speech_to_text(
        self,
        audio_data: bytes,
        request: SpeechToTextRequest,
        raw_request: Request,


sed -n '240,520p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/entrypoints/openai/speech_to_text.py