Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

from dataclasses import dataclass
from typing import Any

from maya_transcribe.align import align_tokens, edit_distance
from maya_transcribe.textnorm import tokenize_words

@dataclass(frozen=True)
class RoverDisagreement:
    kind: str  # "token" | "gap"
    index: int
    chosen: Any
    options: Any

@dataclass(frozen=True)
class RoverResult:
    text: str
    skeleton_key: str
    disagreement_rate: float
    disagreements: list[RoverDisagreement]

def _avg_norm_distance(tokens_a: list[str], tokens_b: list[str]) -> float:
    if not tokens_a and not tokens_b:
        return 0.0
    denom = max(1, (len(tokens_a) + len(tokens_b)) // 2)
    return float(edit_distance(tokens_a, tokens_b)) / float(denom)

def choose_centroid(hyps: dict[str, list[str]]) -> str:
    keys = list(hyps.keys())
    if not keys:
        raise ValueError("No hypotheses provided")
    if len(keys) == 1:
        return keys[0]

    avg: dict[str, float] = {}
    for k in keys:
        dsum = 0.0
        for k2 in keys:
            if k2 == k:
                continue
            dsum += _avg_norm_distance(hyps[k], hyps[k2])
        avg[k] = dsum / float(len(keys) - 1)
    return min(avg.items(), key=lambda kv: kv[1])[0]

def rover_combine(
    hypotheses: dict[str, str],
    weights: dict[str, float] | None = None,
    min_gap_support: float = 0.5,
) -> RoverResult:
    """
    Lightweight ROVER-style combination:
    - pick a centroid hypothesis as skeleton
    - align others to skeleton
    - vote per skeleton token position
    - for gap insertions, keep a centroid insertion sequence if it has >= min_gap_support weight
    """
    if not hypotheses:
        return RoverResult(text="", skeleton_key="", disagreement_rate=0.0, disagreements=[])

    weights = weights or {}
    tokenized: dict[str, list[str]] = {k: tokenize_words(v) for k, v in hypotheses.items()}
    skel_key = choose_centroid(tokenized)
    skel_tokens = tokenized[skel_key]

    keys = list(tokenized.keys())
    total_weight = sum(float(weights.get(k, 1.0)) for k in keys)
    if total_weight <= 0:
        total_weight = float(len(keys))

    aligned_tokens: dict[str, list[str | None]] = {k: [None] * len(skel_tokens) for k in keys}
    gap_inserts: dict[str, list[list[str]]] = {k: [[] for _ in range(len(skel_tokens) + 1)] for k in keys}

    # Skeleton aligns to itself.
    for j, tok in enumerate(skel_tokens):
        aligned_tokens[skel_key][j] = tok

    # Align all others to skeleton.
    for k in keys:
        if k == skel_key:
            continue
        hyp_tokens = tokenized[k]
        alignment = align_tokens(skel_tokens, hyp_tokens).aligned

        skel_pos = 0
        for ref_tok, hyp_tok in alignment:
            if ref_tok is None:
                if hyp_tok is not None:
                    gap_inserts[k][skel_pos].append(hyp_tok)
                continue
            # ref_tok is not None
            if skel_pos >= len(skel_tokens):
                break
            aligned_tokens[k][skel_pos] = hyp_tok
            skel_pos += 1

    disagreements: list[RoverDisagreement] = []

    # Combine gap insertions.
    chosen_gaps: list[list[str]] = [[] for _ in range(len(skel_tokens) + 1)]
    for gap_idx in range(len(skel_tokens) + 1):
        sequences: list[tuple[str, tuple[str, ...], float]] = []
        present_weight = 0.0
        for k in keys:
            w = float(weights.get(k, 1.0))
            seq = tuple(gap_inserts[k][gap_idx])
            if seq:
                present_weight += w
            sequences.append((k, seq, w))

        if present_weight < (min_gap_support * total_weight):
            chosen_gaps[gap_idx] = []
            continue

        # Choose centroid sequence among non-empty candidates (weighted by average distance).
        candidates = [s for s in sequences if s[1]]
        if not candidates:
            chosen_gaps[gap_idx] = []
            continue

        best_seq: tuple[str, ...] | None = None
        best_score: float | None = None
        for _, seq, _w in candidates:
            score = 0.0
            for _, seq2, w2 in candidates:
                score += _avg_norm_distance(list(seq), list(seq2)) * w2
            score /= max(1e-9, present_weight)
            if best_score is None or score < best_score:
                best_score = score
                best_seq = seq

        chosen = list(best_seq or ())
        chosen_gaps[gap_idx] = chosen

        unique = {s[1] for s in candidates}
        if len(unique) > 1:
            disagreements.append(
                RoverDisagreement(
                    kind="gap",
                    index=gap_idx,
                    chosen=chosen,
                    options={k: list(seq) for k, seq, _w in candidates},
                )
            )

    # Combine skeleton positions.
    chosen_tokens: list[str] = []
    for j in range(len(skel_tokens)):
        votes: dict[str, float] = {}
        for k in keys:
            tok = aligned_tokens[k][j]
            if tok is None:
                continue
            tok = tok.strip()
            if not tok:
                continue
            votes[tok] = votes.get(tok, 0.0) + float(weights.get(k, 1.0))

        if not votes:
            continue

        max_vote = max(votes.values())
        top = [t for t, v in votes.items() if v == max_vote]
        if len(top) == 1:
            chosen_tok = top[0]
        else:
            chosen_tok = skel_tokens[j] if skel_tokens[j] in top else sorted(top)[0]

        if len(votes) > 1:
            disagreements.append(
                RoverDisagreement(kind="token", index=j, chosen=chosen_tok, options=dict(sorted(votes.items())))
            )

        chosen_tokens.append(chosen_tok)

    # Assemble final transcript with gap insertions.
    final_tokens: list[str] = []
    for j in range(len(skel_tokens)):
        final_tokens.extend(chosen_gaps[j])
        if j < len(chosen_tokens):
            final_tokens.append(chosen_tokens[j])
    final_tokens.extend(chosen_gaps[len(skel_tokens)])

    text = " ".join(final_tokens).strip()
    total_positions = len(skel_tokens) + (len(skel_tokens) + 1)
    disagreement_rate = float(len(disagreements)) / float(max(1, total_positions))

    return RoverResult(
        text=text,
        skeleton_key=skel_key,
        disagreement_rate=disagreement_rate,
        disagreements=disagreements,
    )


Summary: auto

## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill.
### Available skills
- skill-creator: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Codex's capabilities with specialized knowledge, workflows, or tool integrations. (file: /home/ubuntu/.codex/skills/.system/skill-creator/SKILL.md)
- skill-installer: Install Codex skills into $CODEX_HOME/skills from a curated list or a GitHub repo path. Use when a user asks to list installable skills, install a curated skill, or install a skill from another repo (including private repos). (file: /home/ubuntu/.codex/skills/.system/skill-installer/SKILL.md)
### How to use skills
- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths.
- Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned.
- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback.
- How to use a skill (progressive disclosure):
  1) After deciding to use a skill, open its `SKILL.md`. Read only enough to follow the workflow.
  2) If `SKILL.md` points to extra folders such as `references/`, load only the specific files needed for the request; don't bulk-load everything.
  3) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks.
  4) If `assets/` or templates exist, reuse them instead of recreating from scratch.
- Coordination and sequencing:
  - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them.
  - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why.
- Context hygiene:
  - Keep context small: summarize long sections instead of pasting them; only load extra files when needed.
  - Avoid deep reference-chasing: prefer opening only files directly linked from `SKILL.md` unless you're blocked.
  - When variants exist (frameworks, providers, domains), pick only the relevant reference file(s) and note that choice.
- Safety and fallback: If a skill can't be applied cleanly (missing files, unclear instructions), state the issue, pick the next-best approach, and continue.


**Planning combined results generation**

I'm outlining how to merge segment hypotheses from both models using rover_combine, then create a combined results.jsonl for evaluation, and next will confirm required fields by inspecting eval.py's WER evaluation function.


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from jiwer import cer, wer

from maya_transcribe.io import read_jsonl
from maya_transcribe.textnorm import normalize_text

@dataclass(frozen=True)
class EvalRow:
    segment_id: str
    language: str | None
    decision: str | None
    duration_sec: float | None
    duration_bucket: str | None
    ref: str
    hyp: str
    wer: float
    cer: float
    oracle_best_wer: float | None = None
    oracle_best_cer: float | None = None
    oracle_best_key: str | None = None
    selection_gap_wer: float | None = None

def load_refs(refs_path: Path) -> dict[str, dict[str, Any]]:
    """
    Read a JSONL file of references.
    Expected keys (any one): `text`, `transcript`, `reference`.
    Required key: `segment_id` (or `id`).
    """
    refs: dict[str, dict[str, Any]] = {}
    for row in read_jsonl(refs_path):
        segment_id = str(row.get("segment_id") or row.get("id") or "").strip()
        if not segment_id:
            continue
        refs[segment_id] = row
    return refs

def _get_text(row: dict[str, Any], keys: tuple[str, ...]) -> str | None:
    for k in keys:
        v = row.get(k)
        if isinstance(v, str) and v.strip():
            return v
    return None

def evaluate(
    results_path: Path,
    refs_path: Path,
    out_path: Path | None = None,
    hypothesis: str = "final",
    oracle: bool = True,
    language_source: str = "route",
    decision_filter: list[str] | None = None,
) -> dict[str, Any]:
    refs = load_refs(refs_path)
    results = read_jsonl(results_path)

    decision_filter_set = None
    if decision_filter:
        decision_filter_set = {str(d).strip().lower() for d in decision_filter if str(d).strip()}

    rows: list[EvalRow] = []
    for r in results:
        segment_id = str(r.get("segment_id") or "").strip()
        if not segment_id or segment_id not in refs:
            continue

        ref_row = refs[segment_id]
        ref_text = _get_text(ref_row, ("text", "transcript", "reference"))
        if ref_text is None:
            continue

        hyp_text = extract_hypothesis_text(r, hypothesis=hypothesis)
        if hyp_text is None:
            continue

        route_lang = None
        route = r.get("route") or {}
        if isinstance(route, dict):
            route_lang = route.get("language_hint")

        ref_lang = None
        if language_source == "refs":
            v = ref_row.get("lang") or ref_row.get("language")
            if isinstance(v, str) and v.strip():
                ref_lang = v.strip()
        elif language_source != "route":
            raise ValueError("language_source must be 'route' or 'refs'")

        lang = ref_lang or route_lang

        decision = None
        ver = r.get("verification") or {}
        if isinstance(ver, dict):
            decision = ver.get("decision")
        if decision_filter_set is not None:
            dec_norm = str(decision or "").strip().lower()
            if dec_norm not in decision_filter_set:
                continue

        duration_sec = None
        qc = r.get("qc") or {}
        if isinstance(qc, dict):
            try:
                duration_sec = float(qc.get("duration_sec")) if qc.get("duration_sec") is not None else None
            except (TypeError, ValueError):
                duration_sec = None
        bucket = duration_bucket(duration_sec)

        ref_norm = normalize_text(ref_text, lang=lang)
        hyp_norm = normalize_text(hyp_text, lang=lang)
        w = float(wer(ref_norm, hyp_norm))


Summary: auto


**Planning ensemble evaluation and two-pass proofread**

I'll implement merging of Voxtral base results with Gemma or Vaani updates to create ensemble outputs, using existing pipeline functions or a simple script. Also, I'll modify vLLmTranscriptionASR to accept a prompt and adapt the pipeline for optional two-pass transcription, focusing on Voxtral and supporting ensemble bench evaluation with WER metrics.


**Designing language forcing options for bench runs**

I'll add a `--force-language` CLI option to override route language and bypass LID, updating the pipeline function signature and bench run CLI to accept this parameter. I'll explore how bench invokes the pipeline and threshold handling to properly integrate these language forcing controls.


rg -n "run_pipeline|pipeline" -n maya_transcribe/cli.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
253:    End-to-end pipeline: preprocess -> LID -> ASR ensemble -> ROVER -> verify.
255:    from maya_transcribe.pipeline import run_pipeline
285:    results_path = run_pipeline(
417:    Run the full pipeline on a local benchmark corpus and write eval.json.
422:    from maya_transcribe.pipeline import run_pipeline
463:    results_path = run_pipeline(


sed -n '420,520p' maya_transcribe/cli.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:

    from maya_transcribe.eval import evaluate
    from maya_transcribe.pipeline import run_pipeline

    audio_dir = bench_dir / "audio"
    refs_path = bench_dir / "refs.jsonl"
    if not audio_dir.exists():
        raise typer.BadParameter(f"Missing: {audio_dir}")
    if not refs_path.exists():
        raise typer.BadParameter(f"Missing: {refs_path}")

    if out is None:
        ts = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
        out = Path("outputs/bench") / bench_dir.name / f"run_{ts}"

    ctc_map = None
    if ctc_model_map is not None:
        ctc_map = json.loads(ctc_model_map.read_text(encoding="utf-8"))
        if not isinstance(ctc_map, dict):
            raise typer.BadParameter("ctc_model_map must be a JSON object {lang_code: model_id}")

    tf_map = None
    if transformers_whisper_model_map is not None:
        tf_map = json.loads(transformers_whisper_model_map.read_text(encoding="utf-8"))
        if not isinstance(tf_map, dict):
            raise typer.BadParameter("transformers_whisper_model_map must be a JSON object {lang_code: model_id}")

    ic_map = None
    if indicconformer_model_map is not None:
        ic_map = json.loads(indicconformer_model_map.read_text(encoding="utf-8"))
        if not isinstance(ic_map, dict):
            raise typer.BadParameter("indicconformer_model_map must be a JSON object {lang_code: nemo_model_name}")

    thresholds: dict[str, object] = {"route_lang_from_path": True, "route_lang_prefer_path": True}
    if thresholds_json is not None:
        loaded = json.loads(thresholds_json.read_text(encoding="utf-8"))
        if not isinstance(loaded, dict):
            raise typer.BadParameter("thresholds_json must be a JSON object")
        thresholds.update(loaded)
    if ctc_model or ctc_map:
        thresholds["require_ctc"] = require_ctc
        thresholds["min_ctc_margin"] = float(min_ctc_margin)

    results_path = run_pipeline(
        segments_dir=audio_dir,
        out_dir=out,
        lid_model_id=lid_model_id,
        require_lid_backend=require_lid_backend,
        whisper_model=whisper_model,
        skip_whisper=skip_whisper,
        vllm_asr_model=vllm_asr_model,
        vllm_gpu_memory_utilization=vllm_gpu_memory_utilization,
        vllm_max_model_len=vllm_max_model_len,
        extra_transformers_whisper_models=transformers_whisper_model,
        transformers_whisper_model_map=tf_map,
        mms_asr_model=mms_asr_model,
        indicconformer_model_map=ic_map,
        indicconformer_default_model=indicconformer_default_model,
        ctc_model=ctc_model,
        ctc_model_map=ctc_map,
        trim_silence=trim_silence,
        silence_threshold_db=silence_threshold_db,
        min_silence_sec=min_silence_sec,
        limit=limit,
        thresholds=thresholds,
    )

    eval_path = out / "eval.json"
    summary = evaluate(
        results_path=results_path,
        refs_path=refs_path,
        out_path=eval_path,
        hypothesis=hypothesis,
        oracle=True,
        language_source="refs",
    )
    typer.echo(f"Wrote: {results_path}")
    typer.echo(f"Wrote: {out / 'summary.json'}")
    typer.echo(f"Wrote: {eval_path}")
    typer.echo(json.dumps(summary, ensure_ascii=False, indent=2))

@bench_app.command("push")
def bench_push(
    bench_dir: Path = typer.Option(Path("data/bench/maya_hf_v0"), exists=True, file_okay=False, help="Benchmark corpus directory."),
    repo_id: str = typer.Option(..., help="Hugging Face dataset repo id (e.g., org/name)."),
    private: bool = typer.Option(True, help="Create/push as a private dataset repo."),
    include_audio: bool = typer.Option(False, help="Include audio bytes in the pushed dataset (check dataset licenses!)."),
    split: str = typer.Option("test", help="Split name to use on the Hub."),
) -> None:
    """
    Push a local benchmark corpus to the Hugging Face Hub as a (private) dataset.
    """
    from maya_transcribe.bench import push_benchmark_to_hub

    repo = push_benchmark_to_hub(bench_dir=bench_dir, repo_id=repo_id, private=private, include_audio=include_audio, split=split)
    typer.echo(f"Pushed: {repo}")

if __name__ == "__main__":
    app()


**Confirming route language from path**

The bench run sets route language to prefer path-based inference, which matches observed results showing the route language hint coming from the path despite LID suggesting a different language. This confirms the forced language (hi) is correctly passed for the bench scenario.


**Verifying Voxtral uses route language prompt**

The Voxtral run likely already forces the hi language via route_lang from path in its TranscriptionRequest prompts; I'll confirm this. Next, I'll explore adding an optional prompt parameter for two-pass proofread, focusing on how VllmTranscriptionASR and Gemma3n handle request_prompt.


rg -n "request_prompt" /home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/gemma3n_mm.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
766:        request_prompt: str,


**Planning second-pass prompt integration**

The initial pass ignores request_prompt in gemma3n, so the second pass will focus on Voxtral using its tokenizer and custom encoding. I'll add a new method in VllmTranscriptionASR to build prompts with optional text, leveraging Voxtral's tokenizer logic and checking the model class to apply this only for Voxtral.


**Evaluating prompt usage for transcription correction**

I'm assessing how Voxtral's prompt field can bias transcription, likely by providing previous transcript text without explicit instructions. Full correction instructions might require a chat-based approach with audio, so I'll explore Voxtral's `encode_chat_completion` or related tokenizers to see if audio plus text prompts support correction tasks.


rg -n "encode_.*chat|chat_completion|encode_chat" /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common


Output:
Exit code: 0
Wall time: 0 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/normalize.py:301:    def from_chat_completion_request(self, request: ChatCompletionRequest[UATS]) -> InstructRequestType:
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/normalize.py:319:            >>> instruct_request = normalizer.from_chat_completion_request(request)
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/normalize.py:380:    def from_chat_completion_request(self, request: ChatCompletionRequest[UATS]) -> InstructRequestType:  # type: ignore[type-var, misc]
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/normalize.py:398:            >>> instruct_request = normalizer.from_chat_completion_request(request)
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/mistral.py:124:        self._chat_completion_request_validator = validator
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/mistral.py:139:            self._chat_completion_request_validator._mode,
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/mistral.py:350:    def encode_chat_completion(
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/mistral.py:364:        validated_request = self._chat_completion_request_validator.validate_request(request)
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/mistral.py:374:        instruct_request = self._instruct_request_normalizer.from_chat_completion_request(validated_request)
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/experimental/app/routers.py:50:    tokenized = settings.tokenizer.encode_chat_completion(request)


sed -n '320,420p' /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/mistral.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
                request_normalizer=request_normalizer,
            )
        elif tokenizer.version == TokenizerVersion.v3:
            assert audio_encoder is None, "Tokenizer version needs to be >= v7"
            return MistralTokenizer(
                InstructTokenizerV3(tokenizer, image_encoder=image_encoder),
                validator=MistralRequestValidatorV3(mode=mode),
                request_normalizer=request_normalizer,
            )
        elif tokenizer.version == TokenizerVersion.v7:
            return MistralTokenizer(
                InstructTokenizerV7(tokenizer, image_encoder=image_encoder, audio_encoder=audio_encoder),
                validator=MistralRequestValidatorV5(mode=mode),
                request_normalizer=request_normalizer,
            )
        elif tokenizer.version == TokenizerVersion.v11:
            return MistralTokenizer(
                InstructTokenizerV11(tokenizer, image_encoder=image_encoder, audio_encoder=audio_encoder),
                validator=MistralRequestValidatorV5(mode=mode),
                request_normalizer=request_normalizer,
            )
        elif tokenizer.version == TokenizerVersion.v13:
            return MistralTokenizer(
                InstructTokenizerV13(tokenizer, image_encoder=image_encoder),
                validator=MistralRequestValidatorV13(mode=mode),
                request_normalizer=request_normalizer,
            )

        raise TokenizerException(f"Unrecognized tokenizer filename: {tokenizer_filename}")

    def encode_chat_completion(
        self, request: ChatCompletionRequest[UATS], max_model_input_len: int | None = None
    ) -> TokenizedType:
        r"""Encodes a chat completion request.

        Args:
            request: The chat completion request to encode.
            max_model_input_len: The maximum length of the input to the model.
                If `None`, the input will not be truncated.

        Returns:
            The encoded chat completion request.
        """

        validated_request = self._chat_completion_request_validator.validate_request(request)

        if max_model_input_len is None and request.truncate_for_context_length:
            # the max_model_input_len arg should not be optional ;
            # but this function is used in many small scripts that have no use
            # for truncation, and don't provide the max model len
            raise TokenizerException(
                "encoding a chat completion request with truncation, but no max model len was provided",
            )

        instruct_request = self._instruct_request_normalizer.from_chat_completion_request(validated_request)

        if request.truncate_for_context_length:
            instruct_request.truncate_at_max_tokens = max_model_input_len

        return self.instruct_tokenizer.encode_instruct(instruct_request)

    def encode_transcription(self, request: TranscriptionRequest) -> TokenizedType:
        r"""Encodes a transcription request.

        Args:
            request: The transcription request to encode.

        Returns:
            The encoded transcription request.
        """
        return self.instruct_tokenizer.encode_transcription(request)

    def encode_fim(self, request: FIMRequest) -> TokenizedType:
        r"""Encodes a fill in the middle request.

        Args:
            request: The fill in the middle request to encode.

        Returns:
            The encoded fill in the middle request.
        """
        return self.instruct_tokenizer.encode_fim(request)

    def decode(self, tokens: list[int], special_token_policy: SpecialTokenPolicy | None = None) -> str:
        r"""Decodes a list of tokens into a string.

        Args:
            tokens: The tokens to decode.
            special_token_policy: The policy to use for special tokens. Passing `None` is deprecated and will be changed
                to `SpecialTokenPolicy.IGNORE` in `mistral_common=1.10.0`.

        Returns:
            The decoded string.
        """
        return self.instruct_tokenizer.decode(tokens, special_token_policy=special_token_policy)

    def _to_string(self, tokens: list[int]) -> str:
        return self.instruct_tokenizer._to_string(tokens)

MODEL_NAME_TO_TOKENIZER_CLS: dict[str, Callable[[], MistralTokenizer]] = {


**Exploring audio support in chat completions**

I'm investigating how chat completions handle audio by checking the ChatCompletionRequest type and scanning protocol files for audio-related usage, aiming to understand how to build chat requests with audio included.


rg -n "Audio" /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol -S | head


Output:
Exit code: 0
Wall time: 0 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/transcription/request.py:8:from mistral_common.audio import Audio
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/transcription/request.py:11:from mistral_common.protocol.instruct.chunk import RawAudio
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/transcription/request.py:39:    audio: RawAudio
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/transcription/request.py:81:            audio = Audio.from_base64(self.audio.data)
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/transcription/request.py:129:        audio = Audio.from_bytes(audio_bytes, strict=strict)
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/transcription/request.py:131:        raw_audio = RawAudio(data=audio_str, format=audio.format)
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/messages.py:31:    if name == "AudioURL":
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/messages.py:32:        from mistral_common.protocol.instruct.chunk import AudioURL
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/messages.py:35:        return AudioURL
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/messages.py:36:    elif name == "AudioURLChunk":


sed -n '1,200p' /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/messages.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
import warnings
from enum import Enum
from typing import Any, Literal, TypeVar

from pydantic import Field
from typing_extensions import Annotated, TypeAlias

from mistral_common.base import MistralBase
from mistral_common.protocol.instruct.chunk import (
    ContentChunk,
    TextChunk,
    ThinkChunk,
    UserContentChunk,
    _convert_openai_content_chunks,
)
from mistral_common.protocol.instruct.tool_calls import ToolCall

warnings.filterwarnings(
    action="once",
    category=FutureWarning,
    message=r".*has moved to 'mistral_common.protocol.instruct.chunk'\. It will be removed.*",
)

# TODO(Julien): Remove in 1.10.0
# This is a hacky temporary fix for the deprecation of chunks in message module in favor to the chunk module.
def __getattr__(name: str) -> Any:
    msg = (
        f"{name} has moved to 'mistral_common.protocol.instruct.chunk'. It will be removed from '{__name__}' in 1.10.0."
    )
    if name == "AudioURL":
        from mistral_common.protocol.instruct.chunk import AudioURL

        warnings.warn(msg, FutureWarning)
        return AudioURL
    elif name == "AudioURLChunk":
        from mistral_common.protocol.instruct.chunk import AudioURLChunk

        warnings.warn(msg, FutureWarning)
        return AudioURLChunk
    elif name == "AudioURLType":
        from mistral_common.protocol.instruct.chunk import AudioURLType

        warnings.warn(msg, FutureWarning)
        return AudioURLType
    elif name == "AudioChunk":
        from mistral_common.protocol.instruct.chunk import AudioChunk

        warnings.warn(msg, FutureWarning)
        return AudioChunk
    elif name == "BaseContentChunk":
        from mistral_common.protocol.instruct.chunk import BaseContentChunk

        warnings.warn(msg, FutureWarning)
        return BaseContentChunk
    elif name == "ChunkTypes":
        from mistral_common.protocol.instruct.chunk import ChunkTypes

        warnings.warn(msg, FutureWarning)
        return ChunkTypes
    elif name == "ImageURL":
        from mistral_common.protocol.instruct.chunk import ImageURL

        warnings.warn(msg, FutureWarning)
        return ImageURL
    elif name == "ImageURLChunk":
        from mistral_common.protocol.instruct.chunk import ImageURLChunk

        warnings.warn(msg, FutureWarning)
        return ImageURLChunk
    elif name == "ImageChunk":
        from mistral_common.protocol.instruct.chunk import ImageChunk

        warnings.warn(msg, FutureWarning)
        return ImageChunk
    elif name == "RawAudio":
        from mistral_common.protocol.instruct.chunk import RawAudio

        warnings.warn(msg, FutureWarning)
        return RawAudio
    raise AttributeError(f"module {__name__} has no attribute {name}")

class Roles(str, Enum):
    r"""Enum for the roles of the messages.

    Attributes:
       system: The system role.
       user: The user role.
       assistant: The assistant role.
       tool: The tool role.

    Examples:
        >>> role = Roles.user
    """

    system = "system"
    user = "user"
    assistant = "assistant"
    tool = "tool"

class BaseMessage(MistralBase):
    r"""Base class for all messages.

    Attributes:
       role: The role of the message.
    """

    role: Literal[Roles.system, Roles.user, Roles.assistant, Roles.tool]

    def to_openai(self) -> dict[str, str | list[dict[str, str | dict[str, Any]]]]:
        r"""Converts the message to the OpenAI format.

        Should be implemented by subclasses.
        """
        raise NotImplementedError(f"to_openai method not implemented for {type(self).__name__}")

    @classmethod
    def from_openai(cls, openai_message: dict[str, str | list[dict[str, str | dict[str, Any]]]]) -> "BaseMessage":
        r"""Converts the OpenAI message to the Mistral format.

        Should be implemented by subclasses.
        """
        raise NotImplementedError(f"from_openai method not implemented for {cls.__name__}.")

class UserMessage(BaseMessage):
    r"""User message.

    Attributes:
        content: The content of the message.

    Examples:
        >>> message = UserMessage(content="Can you help me to write a poem?")
    """

    role: Literal[Roles.user] = Roles.user
    content: str | list[UserContentChunk]

    def to_openai(self) -> dict[str, str | list[dict[str, str | dict[str, Any]]]]:
        r"""Converts the message to the OpenAI format."""
        if isinstance(self.content, str):
            return {"role": self.role, "content": self.content}
        return {"role": self.role, "content": [chunk.to_openai() for chunk in self.content]}

    @classmethod
    def from_openai(cls, openai_message: dict[str, str | list[dict[str, str | dict[str, Any]]]]) -> "UserMessage":
        r"""Converts the OpenAI message to the Mistral format."""
        if isinstance(openai_message["content"], str):
            return cls.model_validate(openai_message)
        return cls.model_validate(
            {
                "role": openai_message["role"],
                "content": [_convert_openai_content_chunks(chunk) for chunk in openai_message["content"]],
            },
        )

class SystemMessage(BaseMessage):
    r"""System message.

    Attributes:
        content: The content of the message.

    Examples:
        >>> message = SystemMessage(content="You are a helpful assistant.")
    """

    role: Literal[Roles.system] = Roles.system
    content: str | list[TextChunk | ThinkChunk]

    def to_openai(self) -> dict[str, str | list[dict[str, str | dict[str, Any]]]]:
        r"""Converts the message to the OpenAI format."""
        return self.model_dump()

    @classmethod
    def from_openai(cls, openai_message: dict[str, str | list[dict[str, str | dict[str, Any]]]]) -> "SystemMessage":
        r"""Converts the OpenAI message to the Mistral format."""
        return cls.model_validate(openai_message)

class AssistantMessage(BaseMessage):
    r"""Assistant message.

    Attributes:
        role: The role of the message.
        content: The content of the message.
        tool_calls: The tool calls of the message.
        prefix: Whether the message is a prefix.

    Examples:
        >>> message = AssistantMessage(content="Hello, how can I help you?")
    """

    role: Literal[Roles.assistant] = Roles.assistant
    content: str | list[TextChunk | ThinkChunk] | None = None
    tool_calls: list[ToolCall] | None = None
    prefix: bool = False


**Reviewing audio chunk creation**


sed -n '1,200p' /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/chunk.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
import re
from enum import Enum
from pathlib import Path
from typing import Literal
from urllib.parse import urlparse

from pydantic import ConfigDict, Field, ValidationError, field_validator
from typing_extensions import Annotated

from mistral_common.audio import EXPECTED_FORMAT_VALUES, Audio
from mistral_common.base import MistralBase
from mistral_common.image import SerializableImage

class ChunkTypes(str, Enum):
    r"""Enum for the types of chunks that can be sent to the model.

    Attributes:
       text: A text chunk.
       image: An image chunk.
       image_url: An image url chunk.
       input_audio: An input audio chunk.
       audio_url: An audio url chunk.

    Examples:
        >>> from mistral_common.protocol.instruct.chunk import ChunkTypes
        >>> chunk_type = ChunkTypes.text
    """

    text = "text"
    image = "image"
    image_url = "image_url"
    input_audio = "input_audio"
    audio_url = "audio_url"
    thinking = "thinking"

class BaseContentChunk(MistralBase):
    r"""Base class for all content chunks.

    Content chunks are used to send different types of content to the model.

    Attributes:
       type: The type of the chunk.
    """

    type: Literal[
        ChunkTypes.text,
        ChunkTypes.image,
        ChunkTypes.image_url,
        ChunkTypes.input_audio,
        ChunkTypes.audio_url,
        ChunkTypes.thinking,
    ]

    def to_openai(self) -> dict[str, str | dict[str, str]]:
        r"""Converts the chunk to the OpenAI format.

        Should be implemented by subclasses.
        """
        raise NotImplementedError(f"to_openai method not implemented for {type(self).__name__}")

    @classmethod
    def from_openai(cls, openai_chunk: dict[str, str | dict[str, str]]) -> "BaseContentChunk":
        r"""Converts the OpenAI chunk to the Mistral format.

        Should be implemented by subclasses.
        """
        raise NotImplementedError(f"from_openai method not implemented for {cls.__name__}")

class ImageChunk(BaseContentChunk):
    r"""Image chunk.

    Attributes:
       image: The image to be sent to the model.

    Examples:
        >>> from PIL import Image
        >>> image_chunk = ImageChunk(image=Image.new('RGB', (200, 200), color='blue'))
    """

    type: Literal[ChunkTypes.image] = ChunkTypes.image
    image: SerializableImage
    model_config = ConfigDict(arbitrary_types_allowed=True)

    def to_openai(self) -> dict[str, str | dict[str, str]]:
        r"""Converts the chunk to the OpenAI format."""
        base64_image = self.model_dump(include={"image"}, context={"add_format_prefix": True})["image"]
        return {"type": "image_url", "image_url": {"url": base64_image}}

    @classmethod
    def from_openai(cls, openai_chunk: dict[str, str | dict[str, str]]) -> "ImageChunk":
        r"""Converts the OpenAI chunk to the Mistral format."""
        assert openai_chunk.get("type") == "image_url", openai_chunk

        image_url_dict = openai_chunk["image_url"]
        assert isinstance(image_url_dict, dict) and "url" in image_url_dict, image_url_dict

        if re.match(r"^data:image/\w+;base64,", image_url_dict["url"]):  # Remove the prefix if it exists
            image_url_dict["url"] = image_url_dict["url"].split(",")[1]

        return cls.model_validate({"image": image_url_dict["url"]})

class ImageURL(MistralBase):
    r"""Image URL or a base64 encoded image.

    Attributes:
       url: The URL of the image.
       detail: The detail of the image.

    Examples:
       >>> image_url = ImageURL(url="https://example.com/image.png")
    """

    url: str
    detail: str | None = None

class ImageURLChunk(BaseContentChunk):
    r"""Image URL chunk.

    Attributes:
       image_url: The URL of the image or a base64 encoded image to be sent to the model.

    Examples:
        >>> image_url_chunk = ImageURLChunk(image_url="data:image/png;base64,iVBORw0")
    """

    type: Literal[ChunkTypes.image_url] = ChunkTypes.image_url
    image_url: ImageURL | str

    model_config = ConfigDict(arbitrary_types_allowed=True)

    def get_url(self) -> str:
        if isinstance(self.image_url, ImageURL):
            return self.image_url.url
        return self.image_url

    def to_openai(self) -> dict[str, str | dict[str, str]]:
        r"""Converts the chunk to the OpenAI format."""
        image_url_dict = {"url": self.get_url()}
        if isinstance(self.image_url, ImageURL) and self.image_url.detail is not None:
            image_url_dict["detail"] = self.image_url.detail

        out_dict: dict[str, str | dict[str, str]] = {
            "type": "image_url",
            "image_url": image_url_dict,
        }
        return out_dict

    @classmethod
    def from_openai(cls, openai_chunk: dict[str, str | dict[str, str]]) -> "ImageURLChunk":
        r"""Converts the OpenAI chunk to the Mistral format."""
        return cls.model_validate({"image_url": openai_chunk["image_url"]})

class RawAudio(MistralBase):
    r"""Base64 encoded audio data.

    This class represents raw audio data encoded in base64 format.

    Attributes:
        data: The base64 encoded audio data, which can be a string or bytes.
        format: The format of the audio data.

    Examples:
        >>> audio = RawAudio(data="base64_encoded_audio_data", format="mp3")
    """

    data: str | bytes
    format: str

    @classmethod
    def from_audio(cls, audio: Audio) -> "RawAudio":
        """Creates a RawAudio instance from an Audio object.

        Args:
            audio: An Audio object containing audio data, format, and duration.

        Returns:
            An AudioChunk instance initialized with the audio data.
        """
        format = audio.format
        data = audio.to_base64(format, False)

        return cls(data=data, format=format)

    @field_validator("format")
    def should_not_be_empty(cls, v: str) -> str:
        if v not in EXPECTED_FORMAT_VALUES:
            raise ValidationError(f"`format` should be one of {EXPECTED_FORMAT_VALUES}. Got: {v}`")

        return v

class AudioURL(MistralBase):
    r"""Audio URL.


Summary: auto


sed -n '200,320p' /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/chunk.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:

    Attributes:
        url: The URL of the audio file.
    """

    url: str

class AudioURLType(str, Enum):
    r"""Enum for the types of audio URLs.

    Attributes:
        url: A URL.
        base64: A base64 encoded audio. Can be prefixed with `data:audio/<format>;base64,`.
        file: A file path.
        file_uri: A file URI (eg. `file:///path/to/file`).
    """

    url = "url"
    base64 = "base64"
    file = "file"
    file_uri = "file_uri"

class AudioURLChunk(BaseContentChunk):
    r"""Audio URL chunk.

    Attributes:
        type: The type of the chunk, which is always `ChunkTypes.audio_url`.
        audio_url: The URL of the audio file.
    """

    type: Literal[ChunkTypes.audio_url] = ChunkTypes.audio_url
    audio_url: str | AudioURL

    @property
    def url(self) -> str:
        if isinstance(self.audio_url, AudioURL):
            return self.audio_url.url
        return self.audio_url

    def get_url_type(self) -> AudioURLType:
        r"""Returns the type of the audio URL.

        Note:
            URLs should be either:
            - a valid URL (http:// or https://)
            - a valid file path (e.g. /path/to/file)
            - a valid file URI (e.g. file:///path/to/file)
            - a base64 encoded audio. It is assumed to be base64 encoded if it is not a valid URL or file path.

        Returns:
            The type of the audio URL.
        """
        url_scheme = urlparse(self.url).scheme
        if url_scheme in {"http", "https"}:
            return AudioURLType.url
        elif url_scheme == "data":
            return AudioURLType.base64
        elif url_scheme == "file":
            return AudioURLType.file_uri

        try:
            url_path = Path(self.url)
            exist_path = url_path.exists()
        except OSError:  # File name too long
            exist_path = False

        if exist_path:
            return AudioURLType.file

        return AudioURLType.base64

    def to_openai(self) -> dict[str, str | dict[str, str]]:
        r"""Converts the chunk to the OpenAI format."""
        if isinstance(self.audio_url, AudioURL):
            return self.model_dump()
        else:
            return {"type": self.type, "audio_url": {"url": self.audio_url}}

    @classmethod
    def from_openai(cls, openai_chunk: dict[str, str | dict[str, str]]) -> "AudioURLChunk":
        r"""Converts the OpenAI chunk to the Mistral format."""
        return cls.model_validate(openai_chunk)

class AudioChunk(BaseContentChunk):
    r"""Audio chunk containing raw audio data.

    This class represents a chunk of audio data that can be used as input.

    Attributes:
        type: The type of the chunk, which is always ChunkTypes.input_audio.
        input_audio: The RawAudio object containing the audio data.

    Examples:
        >>> audio_chunk = AudioChunk(input_audio=RawAudio(data="base64_encoded_audio_data", format="mp3"))
    """

    type: Literal[ChunkTypes.input_audio] = ChunkTypes.input_audio
    input_audio: RawAudio

    @field_validator("input_audio")
    def should_not_be_empty(cls, v: RawAudio) -> RawAudio:
        if not v.data.strip():
            raise ValidationError(f"`InputAudio` should not be empty. Got: {v}`")

        return v

    @classmethod
    def from_audio(cls, audio: Audio) -> "AudioChunk":
        r"""Creates an AudioChunk instance from an Audio object.

        Args:
            audio: An Audio object containing audio data.

        Returns:
            An AudioChunk instance initialized with the audio data.
        """
        return cls(input_audio=RawAudio.from_audio(audio))


**Inspecting chat completion request structure**

I'm checking `mistral_common/protocol/chat` to understand how to create a ChatCompletionRequest that can include an audio chunk.


rg -n "class ChatCompletionRequest" /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common -S


Output:
Exit code: 0
Wall time: 0 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/request.py:49:class ChatCompletionRequest(BaseCompletionRequest, Generic[ChatMessageType]):


sed -n '1,200p' /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/protocol/instruct/request.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from enum import Enum
from typing import Any, Generic

from pydantic import Field

from mistral_common.base import MistralBase
from mistral_common.protocol.base import BaseCompletionRequest
from mistral_common.protocol.instruct.converters import (
    _check_openai_fields_names,
    _is_openai_field_name,
    convert_openai_messages,
    convert_openai_tools,
)
from mistral_common.protocol.instruct.messages import (
    ChatMessage,
    ChatMessageType,
)
from mistral_common.protocol.instruct.tool_calls import Tool, ToolChoice, ToolType

class ResponseFormats(str, Enum):
    r"""Enum of the different formats of an instruct response.

    Attributes:
        text: The response is a plain text.
        json: The response is a JSON object.

    Examples:
        >>> response_format = ResponseFormats.text
    """

    text = "text"
    json = "json_object"

class ResponseFormat(MistralBase):
    r"""The format of the response.

    Attributes:
        type: The type of the response.

    Examples:
        >>> response_format = ResponseFormat(type=ResponseFormats.text)
    """

    type: ResponseFormats = ResponseFormats.text

class ChatCompletionRequest(BaseCompletionRequest, Generic[ChatMessageType]):
    r"""Request for a chat completion.

    Attributes:
        model: The model to use for the chat completion.
        messages: The messages to use for the chat completion.
        response_format: The format of the response.
        tools: The tools to use for the chat completion.
        tool_choice: The tool choice to use for the chat completion.
        truncate_for_context_length: Whether to truncate the messages for the context length.
        continue_final_message: Whether to continue the final message.

    Examples:
        >>> from mistral_common.protocol.instruct.messages import UserMessage, AssistantMessage
        >>> from mistral_common.protocol.instruct.tool_calls import ToolTypes, Function
        >>> request = ChatCompletionRequest(
        ...     messages=[
        ...         UserMessage(content="Hello!"),
        ...         AssistantMessage(content="Hi! How can I help you?"),
        ...     ],
        ...     response_format=ResponseFormat(type=ResponseFormats.text),
        ...     tools=[Tool(type=ToolTypes.function, function=Function(name="get_weather", parameters={}))],
        ...     tool_choice=ToolChoice.auto,
        ...     truncate_for_context_length=True,
        ... )
    """

    model: str | None = None
    messages: list[ChatMessageType]
    response_format: ResponseFormat = Field(default_factory=ResponseFormat)
    tools: list[Tool] | None = None
    tool_choice: ToolChoice = ToolChoice.auto
    truncate_for_context_length: bool = False
    continue_final_message: bool = False

    def to_openai(self, **kwargs: Any) -> dict[str, list[dict[str, Any]]]:
        r"""Convert the request messages and tools into the OpenAI format.

        Args:
            kwargs: Additional parameters to be added to the request.

        Returns:
            The request in the OpenAI format.

        Examples:
            >>> from mistral_common.protocol.instruct.messages import UserMessage
            >>> from mistral_common.protocol.instruct.tool_calls import Tool, Function
            >>> request = ChatCompletionRequest(messages=[UserMessage(content="Hello, how are you?")], temperature=0.15)
            >>> request.to_openai(stream=True)
            {'temperature': 0.15, 'top_p': 1.0, 'response_format': {'type': 'text'}, 'tool_choice': 'auto', 'continue_final_message': False, 'messages': [{'role': 'user', 'content': 'Hello, how are you?'}], 'stream': True}
            >>> request = ChatCompletionRequest(messages=[UserMessage(content="Hello, how are you?")], tools=[
            ...     Tool(function=Function(
            ...         name="get_current_weather",
            ...         description="Get the current weather in a given location",
            ...         parameters={
            ...             "type": "object",
            ...             "properties": {
            ...                 "location": {
            ...                     "type": "string",
            ...                     "description": "The city and state, e.g. San Francisco, CA",
            ...                 },
            ...                 "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
            ...             },
            ...             "required": ["location"],
            ...         },
            ...     ),
            ... )])
            >>> request.to_openai()
            {'temperature': 0.7, 'top_p': 1.0, 'response_format': {'type': 'text'}, 'tool_choice': 'auto', 'continue_final_message': False, 'messages': [{'role': 'user', 'content': 'Hello, how are you?'}], 'tools': [{'type': 'function', 'function': {'name': 'get_current_weather', 'description': 'Get the current weather in a given location', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city and state, e.g. San Francisco, CA'}, 'unit': {'type': 'string', 'enum': ['celsius', 'fahrenheit']}}, 'required': ['location']}}}]}
        """  # noqa: E501

        # Handle messages and tools separately.
        openai_request: dict[str, Any] = self.model_dump(
            exclude={"messages", "tools", "truncate_for_context_length"}, exclude_none=True
        )

        # Rename random_seed to seed.
        seed = openai_request.pop("random_seed", None)
        if seed is not None:
            openai_request["seed"] = seed

        if self.truncate_for_context_length:
            raise NotImplementedError("Truncating for context length is not implemented for OpenAI requests.")

        for kwarg in kwargs:
            # Check for duplicate keyword arguments.
            if kwarg in openai_request:
                raise ValueError(f"Duplicate keyword argument: {kwarg}")
            # Check if kwarg should have been set in the request.
            # This occurs when the field is different between the Mistral and OpenAI API.
            elif kwarg in ChatCompletionRequest.model_fields:
                raise ValueError(f"Keyword argument {kwarg} is already set in the request.")
            # Check if kwarg is a valid OpenAI field name.
            elif not _is_openai_field_name(kwarg):
                raise ValueError(f"Invalid keyword argument: {kwarg}, it should be an OpenAI field name.")

        openai_messages = []
        for message in self.messages:
            openai_messages.append(message.to_openai())

        openai_request["messages"] = openai_messages
        if self.tools is not None:
            openai_request["tools"] = [tool.to_openai() for tool in self.tools]

        openai_request.update(kwargs)

        return openai_request

    @classmethod
    def from_openai(
        cls,
        messages: list[dict[str, str | list[dict[str, str | dict[str, Any]]]]],
        tools: list[dict[str, Any]] | None = None,
        continue_final_message: bool = False,
        **kwargs: Any,
    ) -> "ChatCompletionRequest":
        r"""Create a chat completion request from the OpenAI format.

        Args:
            messages: The messages in the OpenAI format.
            tools: The tools in the OpenAI format.
            continue_final_message: Whether to continue the final message.
            **kwargs: Additional keyword arguments to pass to the constructor. These should be the same as the fields
                of the request class or the OpenAI API equivalent.

        Returns:
            The chat completion request.
        """
        if "seed" in kwargs and "random_seed" in kwargs:
            raise ValueError("Cannot specify both `seed` and `random_seed`.")

        random_seed = kwargs.pop("seed", None) or kwargs.pop("random_seed", None)

        _check_openai_fields_names(set(cls.model_fields.keys()), set(kwargs.keys()))

        converted_messages: list[ChatMessage] = convert_openai_messages(messages)

        converted_tools = convert_openai_tools(tools) if tools is not None else None

        return cls(
            messages=converted_messages,  # type: ignore[arg-type]
            tools=converted_tools,
            random_seed=random_seed,
            continue_final_message=continue_final_message,
            **kwargs,
        )

class InstructRequest(MistralBase, Generic[ChatMessageType, ToolType]):
    r"""A valid Instruct request to be tokenized.


**Inspecting audio chunk creation**

I'm checking how to build an AudioChunk from an Audio object, noting that mistral_common.audio.Audio expects audio bytes but considering if numpy input can be converted conveniently. Next, I'll review audio.py for relevant constructors or helpers.


sed -n '1,200p' /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/audio.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
import base64
import io
import logging
import re
from enum import Enum
from functools import cache
from pathlib import Path
from typing import TYPE_CHECKING

import numpy as np
import requests

from mistral_common.imports import (
    assert_soundfile_installed,
    assert_soxr_installed,
    is_soundfile_installed,
    is_soxr_installed,
)

if TYPE_CHECKING:
    from mistral_common.protocol.instruct.chunk import RawAudio

logger = logging.getLogger(__name__)

if is_soundfile_installed():
    import soundfile as sf

    # Get the available formats from soundfile
    available_formats = sf.available_formats()

    # Create an Enum dynamically
    AudioFormat = Enum("AudioFormat", {format_name: format_name for format_name in available_formats})  # type: ignore[misc]
else:
    AudioFormat = Enum("AudioFormat", {"none": "none"})  # type: ignore[no-redef]

if is_soxr_installed():
    import soxr

EXPECTED_FORMAT_VALUES = [v.value.lower() for v in AudioFormat.__members__.values()]

class Audio:
    def __init__(self, audio_array: np.ndarray, sampling_rate: int, format: str) -> None:
        r"""Initialize an Audio instance with audio data, sampling rate, and format.

        Args:
            audio_array: The audio data as a numpy array.
            sampling_rate: The sampling rate of the audio in Hz.
            format: The format of the audio file.
        """
        self.audio_array = audio_array
        self.sampling_rate = sampling_rate
        self.format = format
        self._check_valid()

    def __repr__(self) -> str:
        return (
            f"Audio - sampling_rate={self.sampling_rate} Hz, "
            f"duration={len(self.audio_array) / self.sampling_rate:.2f}s, "
            f"shape={self.audio_array.shape}"
        )

    def _check_valid(self) -> None:
        assert isinstance(self.audio_array, np.ndarray), type(np.ndarray)
        assert self.audio_array.ndim == 1, f"{self.audio_array.ndim=}"
        assert_soundfile_installed()
        assert self.format in EXPECTED_FORMAT_VALUES, f"{self.format=} not in {EXPECTED_FORMAT_VALUES=}"

    @property
    def duration(self) -> float:
        r"""Calculate the duration of the audio in seconds.

        Returns:
           The duration of the audio in seconds.
        """
        # in seconds
        duration: float = self.audio_array.shape[0] / self.sampling_rate
        return duration

    @staticmethod
    def from_url(url: str, strict: bool = True) -> "Audio":
        r"""Create an Audio instance from a URL.

        Args:
            url: The URL of the audio file.
            strict: Whether to strictly enforce mono audio.

        Returns:
            An instance of the Audio class.
        """
        try:
            response = requests.get(url)
            response.raise_for_status()
            return Audio.from_bytes(response.content, strict=strict)
        except requests.RequestException as e:  # Something went wrong with the request.
            raise ValueError(f"Failed to download audio from URL: {url}") from e
        except Exception as e:  # Something went wrong with the audio file.
            raise ValueError(f"Failed to create Audio instance from URL: {url} .") from e

    @staticmethod
    def from_base64(audio_base64: str, strict: bool = True) -> "Audio":
        r"""Create an Audio instance from a base64 encoded string.

        Args:
            audio_base64: The base64 encoded audio data.
            strict: Whether to strictly enforce mono audio. Defaults to True.

        Returns:
            An instance of the Audio class.
        """
        assert_soundfile_installed()

        if re.match(r"^data:audio/\w+;base64,", audio_base64):  # Remove the prefix if it exists
            audio_base64 = audio_base64.split(",")[1]

        try:
            audio_bytes = base64.b64decode(audio_base64)
        except Exception as e:
            raise ValueError("base64 decoding failed. Please check the input string is a valid base64.") from e

        return Audio.from_bytes(audio_bytes, strict=strict)

    @staticmethod
    def from_file(file: str, strict: bool = True) -> "Audio":
        r"""Create an Audio instance from an audio file.

        Args:
            file: Path to the audio file.
            strict: Whether to strictly enforce mono audio. Defaults to True.

        Returns:
            An instance of the Audio class.
        """
        assert_soundfile_installed()

        if isinstance(file, str) and file.startswith("file://"):
            file = file[7:]

        if not Path(file).exists():
            raise FileNotFoundError(f"{file=} does not exist")

        with open(file, "rb") as f:
            audio_bytes = f.read()

        return Audio.from_bytes(audio_bytes, strict=strict)

    @staticmethod
    def from_bytes(audio_bytes: bytes, strict: bool = True) -> "Audio":
        r"""Create an Audio instance from bytes.

        Args:
            audio_bytes: The audio data as bytes.
            strict: Whether to strictly enforce mono audio. Defaults to True.

        Returns:
            An instance of the Audio class.
        """
        # Read the bytes into an audio file.
        with io.BytesIO(audio_bytes) as audio_file:
            with sf.SoundFile(audio_file) as f:
                # Read the entire audio data
                audio_array = f.read(dtype="float32")
                sampling_rate = f.samplerate
                audio_format = f.format

        format_enum = AudioFormat(audio_format)
        format = format_enum.value.lower()

        if audio_array.ndim != 1:
            if strict:
                raise ValueError(f"{audio_array.ndim=}")
            else:
                audio_array = audio_array.mean(axis=1)

        return Audio(audio_array=audio_array, sampling_rate=sampling_rate, format=format)

    def to_base64(self, format: str, prefix: bool = False) -> str:
        r"""Convert the audio data to a base64 encoded string.

        Args:
            format: The format to encode the audio in.
            prefix: Whether to add a data prefix to the base64 encoded string.

        Returns:
            The base64 encoded audio data.
        """
        assert_soundfile_installed()

        assert format in EXPECTED_FORMAT_VALUES, f"{format=} not in {EXPECTED_FORMAT_VALUES=}"

        with io.BytesIO() as audio_file:
            sf.write(audio_file, self.audio_array, self.sampling_rate, format=format.upper())
            audio_file.seek(0)
            base64_str = base64.b64encode(audio_file.read()).decode("utf-8")
        if prefix:
            base64_str = f"data:audio/{format.lower()};base64,{base64_str}"
        return base64_str

    @staticmethod
    def from_raw_audio(audio: "RawAudio") -> "Audio":


**Evaluating audio plus transcript prompt approach**

Considering using the first-pass transcript as a prompt for a second-pass transcription, but transcription requests require the prompt to match audio language and don’t handle extra instructions well; this suggests separate handling of instructions or leveraging chat completion with audio, though model support for audio in chat remains uncertain.


**Planning custom prompt injection for Voxtral transcription**

I'm mapping out adding support for injecting a Hindi instruction prompt alongside the first-pass transcript into Voxtral's transcription calls by customizing prompt construction outside the existing model method, avoiding multiple site-package patches while ensuring compatibility with Voxtral's expected TranscriptionRequest format.


sed -n '1,120p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import math
from collections.abc import Iterable, Mapping, Sequence
from functools import cached_property
from math import ceil
from typing import Literal, cast

import numpy as np
import regex as re
import torch
import torch.nn as nn
from mistral_common.audio import mel_filter_bank
from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
from mistral_common.protocol.transcription.request import TranscriptionRequest
from mistral_common.tokens.tokenizers.audio import Audio, AudioEncoder
from transformers import BatchFeature, TensorType, WhisperConfig
from transformers.tokenization_utils_base import TextInput

from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs.data import PromptType
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models import SupportsPP
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.whisper import WhisperEncoder
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
    MultiModalDataDict,
    MultiModalFieldConfig,
    MultiModalKwargsItems,
    MultiModalUUIDDict,
    NestedTensors,
)
from vllm.multimodal.parse import (
    AudioProcessorItems,
    MultiModalDataItems,
    MultiModalDataParser,
)
from vllm.multimodal.processing import (
    BaseMultiModalProcessor,
    BaseProcessingInfo,
    MultiModalProcessingInfo,
    PromptReplacement,
    PromptUpdate,
)
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.mistral import MistralTokenizer

from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
from .utils import init_vllm_registered_model, maybe_prefix

logger = init_logger(__name__)

ISO639_1_SUPPORTED_LANGS = {
    "ar": "Arabic",
    "nl": "Dutch",
    "en": "English",
    "fr": "French",
    "de": "German",
    "hi": "Hindi",
    "it": "Italian",
    "pt": "Portuguese",
    "es": "Spanish",
}

class VoxtralProcessorAdapter:
    """
    Provide a HF-compatible interface for
    :class:`mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
    """

    def __init__(self, tokenizer: MistralTokenizer) -> None:
        super().__init__()
        self.tokenizer = tokenizer

    @cached_property
    def _audio_processor(self) -> AudioEncoder:
        audio_encoder = self.tokenizer.instruct.audio_encoder
        assert isinstance(audio_encoder, AudioEncoder)
        return audio_encoder

    @cached_property
    def audio_token_id(self) -> int:
        return self._audio_processor.special_ids.audio

    @cached_property
    def begin_audio_token_id(self) -> int:
        return self._audio_processor.special_ids.begin_audio

    # @cached_property
    # def begin_transcript_token_id(self) -> int:
    #     return self._audio_processor.special_ids.begin_transcript

    # @cached_property
    # def end_transcript_token_id(self) -> int:
    #     return self._audio_processor.special_ids.end_transcript

    @cached_property
    def sampling_rate(self) -> int:
        return self._audio_processor.audio_config.sampling_rate

    @cached_property
    def frame_rate(self) -> float:
        return self._audio_processor.audio_config.frame_rate

    def get_num_audio_tokens(
        self,
        audio_length: int,
    ) -> int:
        pad_audio_length = self._audio_processor.next_multiple_of_chunk_frames(
            audio_length, self.sampling_rate


**Searching for chat encoding method**


rg -n "chat" /home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py | head -n 40


Output:
Exit code: 0
Wall time: 0 seconds
Output:
149:                "tokenizer or pass a chat completion request. "
262:        res = tokenizer.mistral.encode_chat_completion(request)
321:        # NOTE: The tokens are already inserted by the chat template


sed -n '220,340p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
        mm_options: Mapping[str, BaseDummyOptions] | None = None,
    ) -> MultiModalDataDict:
        num_audios = mm_counts.get("audio", 0)

        target_length = self.info.get_max_audio_array_len()

        audio_overrides = mm_options.get("audio") if mm_options else None

        return {
            "audio": self._get_dummy_audios(
                length=target_length, num_audios=num_audios, overrides=audio_overrides
            )
        }

    def get_dummy_processor_inputs(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
        mm_options: Mapping[str, BaseDummyOptions] | None = None,
    ) -> ProcessorInputs:
        tokenizer = self.info.get_tokenizer()

        dummy_text = self.get_dummy_text(mm_counts)
        dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
        dummy_audios = dummy_mm_data.get("audio", [])

        audio_chunks: list[AudioChunk] = []
        format = "wav"
        for audio in dummy_audios:
            audio_item = Audio(
                audio_array=audio,
                sampling_rate=self.info.get_hf_processor().sampling_rate,
                format=format,
            )
            chunk = AudioChunk(input_audio=RawAudio.from_audio(audio_item))
            audio_chunks.append(chunk)

        request = ChatCompletionRequest(
            messages=[
                UserMessage(content=[TextChunk(text=dummy_text), *audio_chunks]),
            ]
        )
        res = tokenizer.mistral.encode_chat_completion(request)
        dummy_tokens = res.tokens
        # whixtral tokenizer adds padding to the audio
        # so we need to update the audio arrays
        dummy_mm_data["audio"] = [a.audio_array for a in res.audios]

        return ProcessorInputs(prompt=dummy_tokens, mm_data=dummy_mm_data)

class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
    def _get_mm_fields_config(
        self,
        hf_inputs: Mapping[str, NestedTensors],
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        return dict(audio_arrays=MultiModalFieldConfig.batched("audio"))

    def _get_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> Sequence[PromptUpdate]:
        processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

        audio_id = processor.audio_token_id

        def get_replacement(item_idx: int):
            audios = mm_items.get_items("audio", AudioProcessorItems)
            audio_len = audios.get_audio_length(item_idx)

            nb_audio_tokens = processor.get_num_audio_tokens(audio_len)

            return [audio_id] * nb_audio_tokens

        return [
            PromptReplacement(
                modality="audio",
                target="",  # Never match the prompt (see below note)
                replacement=get_replacement,
            ),
        ]

    def _cached_apply_hf_processor(
        self,
        prompt: str | list[int],
        mm_data_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        tokenization_kwargs: Mapping[str, object],
        mm_uuids: MultiModalUUIDDict | None = None,
    ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
        prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
            prompt=prompt,
            mm_data_items=mm_data_items,
            hf_processor_mm_kwargs=hf_processor_mm_kwargs,
            tokenization_kwargs=tokenization_kwargs,
            mm_uuids=mm_uuids,
        )

        # NOTE: The tokens are already inserted by the chat template
        return prompt_ids, mm_info, True

    def _get_data_parser(self) -> MultiModalDataParser:
        sampling_rate = self.info.get_hf_processor().sampling_rate
        return MultiModalDataParser(target_sr=sampling_rate)

@MULTIMODAL_REGISTRY.register_processor(
    VoxtralMultiModalProcessor,
    info=VoxtralProcessingInfo,
    dummy_inputs=VoxtralDummyInputsBuilder,
)
class VoxtralForConditionalGeneration(
    nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, SupportsTranscription
):
    supported_languages = ISO639_1_SUPPORTED_LANGS

    packed_modules_mapping = {
        "qkv_proj": ["q_proj", "k_proj", "v_proj"],


**Planning 2-pass chat input with audio**

The approach is to build a chat completion request combining text instructions and audio chunk (with audio after text), encode it via the tokenizer to get prompt tokens and audio arrays, then run the model generate call with these inputs for pass 2; this aligns with existing dummy input structure and should leverage the chat template effectively.


**Outlining multi-step Voxtral 2-pass proofread integration**

I'm planning to extend the ASR pipeline with a configurable second pass "proofread" for Voxtral models using chat prompts with audio and prior transcript, adding CLI options and pipeline flags to enable it, and handling ensemble merging via a separate script; the transcription method will be adapted to accept optional prompts and modes, with Voxtral-specific logic to build appropriate requests for multi-modal generation.


rg -n "encode_transcription" /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common -S


Output:
Exit code: 0
Wall time: 0 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/mistral.py:381:    def encode_transcription(self, request: TranscriptionRequest) -> TokenizedType:
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/mistral.py:390:        return self.instruct_tokenizer.encode_transcription(request)
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/instruct.py:377:    def encode_transcription(self, request: TranscriptionRequest) -> Tokenized:
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/instruct.py:937:    def encode_transcription(self, request: TranscriptionRequest) -> Tokenized:
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/base.py:318:    def encode_transcription(self, request: TranscriptionRequest) -> TokenizedType:


sed -n '880,980p' /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/instruct.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
        Args:
            content: The content to encode.
            is_last: Whether the message is the last one.
            system_prompt: The system prompt.
            force_img_first: Whether to force the image to be first.

        Returns:
            The encoded tokens and the images.
        """
        assert system_prompt is None, "in Tokenizer V7 we don't encode system prompts in user messages"

        if isinstance(content, str):
            return super().encode_user_content(content, is_last, system_prompt)

        has_one_img_one_text_first = len(content) == 2 and isinstance(content[1], (ImageChunk, ImageURLChunk))
        if force_img_first and has_one_img_one_text_first:
            # make sure that if exactly one image and text chunk are passed we force the image chunk to be first
            content = [content[1], content[0]]

        tokens, images, audio = self._encode_content_chunks(content)
        return tokens, images, audio

    def encode_user_message(
        self,
        message: UserMessage,
        available_tools: list[Tool] | None,
        is_last: bool,
        is_first: bool,
        system_prompt: str | None = None,
        force_img_first: bool = False,
    ) -> tuple[list[int], list[np.ndarray], list[Audio]]:
        r"""Encode a user message.

        Args:
            message: The message to encode.
            available_tools: The list of available tools if any.
            is_last: Whether the message is the last one.
            is_first: Whether the message is the first one.
            system_prompt: Not used.
            force_img_first: Whether to force the image to be first.

        Returns:
            The encoded tokens and the list of images.
        """
        assert system_prompt is None, "in Tokenizer V7 we don't encode system prompts in user messages"

        tokens, images, audio = super().encode_user_message(
            message,
            available_tools,
            is_last=is_last,
            is_first=is_first,
            system_prompt=None,
            force_img_first=force_img_first,
        )

        return tokens, images, audio

    def encode_transcription(self, request: TranscriptionRequest) -> Tokenized:
        r"""
        Encodes an audio transcription request into a tokenized format.

        This method processes a transcription request containing audio data,
        encodes the user message, and returns the tokenized output.

        Args:
            request: The transcription request object containing
                the audio data to be encoded.

        Returns:
            Tokenized: The tokenized representation of the audio data, including processed audio and tokens
        """
        assert self.audio_encoder is not None, f"Audio encoder must be defined, got {self.audio_encoder=}"
        if self.audio_encoder.audio_config.transcription_format == TranscriptionFormat.INSTRUCT:
            return self._encode_instruct_transcription(request)
        elif self.audio_encoder.audio_config.transcription_format == TranscriptionFormat.STREAMING:
            return self._encode_streaming_transcription(request)

        raise InvalidRequestException(
            "Transcription format should be one of 'instruct', 'streaming', got "
            f"{self.audio_encoder.audio_config.transcription_format=}."
        )

    def _encode_instruct_transcription(self, request: TranscriptionRequest) -> Tokenized:
        assert request.streaming == StreamingMode.DISABLED, (
            f"Request must not be in streaming mode, got {request.streaming=}"
        )
        assert self.TRANSCRIBE is not None, f"{self.__class__.__name__} needs to have a TRANSCRIBE token"
        prefix = self.start()
        tokens, _, audio = self.encode_user_message(
            UserMessage(content=[AudioChunk(input_audio=request.audio)]),
            available_tools=[],
            is_last=True,
            is_first=True,
            system_prompt=None,
        )

        tokens = [*prefix, *tokens]
        if request.language is not None:
            language_string = f"lang:{request.language}"  # no space.
            tokens += self.tokenizer.encode(language_string, bos=False, eos=False)


Summary: auto


sed -n '980,1080p' /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/instruct.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:

        tokens.append(self.TRANSCRIBE)
        return Tokenized(tokens=tokens, text=self.tokenizer._to_string(tokens), audios=audio)

    def _encode_audio(self, audio: str | bytes, is_online_streaming: bool) -> Tokenized:
        assert self.audio_encoder is not None, (
            f"Audio encoder must be defined to encode audio, got {self.audio_encoder=}"
        )
        _audio = Audio.from_base64(audio) if isinstance(audio, str) else Audio.from_bytes(audio)
        audio_enc = self.audio_encoder.encode_audio(_audio, is_online_streaming=is_online_streaming)

        return Tokenized(
            tokens=audio_enc.tokens,
            audios=[audio_enc.audio],
        )

    def _encode_streaming_transcription(self, request: TranscriptionRequest) -> Tokenized:
        assert request.streaming != StreamingMode.DISABLED, (
            f"Request must be in streaming mode, got {request.streaming=}"
        )

        tokenized_audio = self._encode_audio(
            request.audio.data, is_online_streaming=request.streaming == StreamingMode.ONLINE
        )

        assert isinstance(self.audio_encoder, AudioEncoder), f"Audio encoder must be defined, got {self.audio_encoder=}"
        assert isinstance(self.audio_encoder.audio_config.encoding_config, AudioSpectrogramConfig), (
            f"Audio encoder must be spectrogram encoder, got {self.audio_encoder=}"
        )
        assert self.audio_encoder.audio_config.transcription_delay_ms is not None

        # streaming pad tokens
        tokens = self.start() + [self.STREAMING_PAD] * self.audio_encoder.audio_config.num_delay_tokens

        tokenized = Tokenized(
            tokens=tokens,
            text=self.decode(tokens, special_token_policy=SpecialTokenPolicy.KEEP),
            audios=tokenized_audio.audios,
        )
        return tokenized

    @classmethod
    def validate_messages(cls, messages: list[UATS]) -> None:
        if cls._has_audio(messages):
            if any(isinstance(message, SystemMessage) for message in messages):
                raise ValueError("System messages are not yet allowed when audio is present")

    @staticmethod
    def _has_audio(messages: list[UATS]) -> bool:
        return any(
            isinstance(message, UserMessage)
            and isinstance(message.content, list)
            and any(isinstance(chunk, AudioChunk) for chunk in message.content)
            for message in messages
        )

    def encode_tool_message(self, message: ToolMessage, is_before_last_user_message: bool) -> list[int]:
        r"""Encode a tool message.

        Note:
            Same as [V3][mistral_common.tokens.tokenizers.instruct.InstructTokenizerV3.encode_tool_message]
            but tools are not wrapped in a list and history is also tokenized

        Args:
            message: The message to encode.
            is_before_last_user_message: Not used.

        Returns:
            The encoded tokens.
        """
        assert message.tool_call_id is not None
        assert isinstance(message.content, str), "Message content must be normalized"
        tool_call_id_tokens = self.tokenizer.encode(message.tool_call_id, bos=False, eos=False)
        tokens = self.tokenizer.encode(message.content, bos=False, eos=False)

        prefix_tokens = [
            self.BEGIN_TOOL_RESULTS,
            *tool_call_id_tokens,
            self.BEGIN_TOOL_CONTENT,
        ]
        curr_tokens = [
            *prefix_tokens,
            *tokens,
            self.END_TOOL_RESULTS,
        ]
        return curr_tokens

    def encode_assistant_message(
        self, message: AssistantMessageType, is_before_last_user_message: bool, continue_message: bool
    ) -> list[int]:
        r"""Encode an assistant message.

        Args:
            message: The message to encode.
            is_before_last_user_message: Not used.
            continue_message: Whether to continue the message generation.
                Only use this if the assistant message is the last message.

        Returns:
            The encoded tokens.
        """


**Planning chat completion with audio and text**

To include instructions for pass2, I'll implement chat completion using a UserMessage combining text (instruction and previous transcript) and audio chunks, as system messages aren't allowed with audio. I'll follow existing patterns by placing text before audio and ensure encoding supports this multipart content for feeding into the model.


**Defining audio-text prompt builder**

I'm drafting a helper to build a chat prompt combining text and audio using the tokenizer's encode_chat_completion, replicating Voxtral's padding behavior by extracting processed audio arrays for multi-modal input. Next, I'll confirm the Tokenized type to ensure correct audio array handling before implementation.


rg -n "class Tokenized" /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers -S


Output:
Exit code: 0
Wall time: 0 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/base.py:169:class Tokenized(MistralBase):


sed -n '160,240p' /home/ubuntu/.local/lib/python3.10/site-packages/mistral_common/tokens/tokenizers/base.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:

    v1 = "v1"  # vocab_size = 32000
    v2 = "v2"  # vocab_size = 32768 with special control tokens [INST], [\INST]
    v3 = "v3"  # vocab_size = 32768 (spm) OR 131072 (tekken) with improved function calling
    v7 = "v7"  # vocab_size = 32768 (spm) or 131072 (tekken) with improved system prompt and function calling
    v11 = "v11"  # 131072 (tekken) with improved function calling
    v13 = "v13"  # 131072 (tekken) with no call id and better prompt caching

class Tokenized(MistralBase):
    r"""A tokenized [`InstructRequest`][mistral_common.tokens.instruct.request].

    Attributes:
        tokens: The token ids.
        text: The text representation of the tokens.
        prefix_ids: The prefix ids for FIM.
        images: The loaded images associated with the tokens.

    Examples:
        >>> tokenized = Tokenized(tokens=[1, 2, 3], text="Hello world", prefix_ids=[1], images=[])
    """

    model_config = ConfigDict(arbitrary_types_allowed=True)
    tokens: list[int]
    text: str | None = None
    prefix_ids: list[int] | None = None
    images: list[np.ndarray] = Field(default_factory=list)
    audios: list[Audio] = Field(default_factory=list)

class Tokenizer(ABC):
    @property
    @abstractmethod
    def n_words(self) -> int:
        r"""Vocabulary size of the tokenizer."""

    @abstractmethod
    def vocab(self) -> list[str]:
        r"""All tokens in the vocabulary as strings."""

    @abstractmethod
    def id_to_piece(self, token_id: int) -> str:
        r"""Convert a token id to the token str."""

    @property
    @abstractmethod
    def bos_id(self) -> int:
        r"""id of the Beginning of String token."""

    @property
    @abstractmethod
    def eos_id(self) -> int:
        r"""id of the End of String token."""

    @property
    @abstractmethod
    def pad_id(self) -> int:
        r"""id of the Pad token."""

    @property
    @abstractmethod
    def unk_id(self) -> int:
        r"""id of the Unk token."""

    @abstractmethod
    def encode(self, s: str, bos: bool, eos: bool) -> list[int]:
        """Convert a string to a list of token ids."""

    @abstractmethod
    def decode(self, tokens: list[int], special_token_policy: SpecialTokenPolicy | None = None) -> str:
        r"""Decode the token ids to a string.

        Args:
            tokens: The token ids to decode.
            special_token_policy: The policy to use for special tokens.
                Passing `None` will default to `self._special_token_policy` for
                [Tekkenizer][mistral_common.tokens.tokenizers.tekken.Tekkenizer] and `SpecialTokenPolicy.IGNORE`
                for [SentencePieceTokenizer][mistral_common.tokens.tokenizers.sentencepiece.SentencePieceTokenizer].
                Note that passing `None` will be deprecated and `special_token_policy` will default to
                `SpecialTokenPolicy.IGNORE` in `mistral_common=1.10.0`.


**Designing Voxtral prompt handling**

Planning to implement a `_voxtral_prompt_from_request` method that follows Voxtral’s pattern of packaging the first tokenized audio array with the config sample rate for multi-modal input, ensuring compatibility with the patched padding behavior and model ID checks for Voxtral-specific logic.


**Designing two-pass proofread integration**

Planning to add a _build_prompt method supporting Voxtral chat prompts in proofreading mode, plus a second pass in the pipeline with proofread-specific prompts and options to enable/disable it and prefer proofread results as final. I'll store both passes in ASR results with distinct keys, letting evaluation tools target proofread output separately while optionally overriding base results for final consensus.


rg -n "class ASRResult" -S maya_transcribe/asr.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
26:class ASRResult:


Output:
Exit code: 0
Wall time: 0 seconds
Output:
from __future__ import annotations

import gc
from collections import OrderedDict
from dataclasses import dataclass
from pathlib import Path
from typing import Any

from maya_transcribe.language import ISO3_TO_WHISPER_LANG, to_whisper_lang

_WHISPER_TO_ISO3: dict[str, str] = {v: k for k, v in ISO3_TO_WHISPER_LANG.items()}

@dataclass(frozen=True)
class ASRSegment:
    start: float
    end: float
    text: str
    avg_logprob: float | None = None
    no_speech_prob: float | None = None
    compression_ratio: float | None = None

@dataclass(frozen=True)
class ASRResult:
    model: str
    text: str
    language: str | None
    language_probability: float | None
    segments: list[ASRSegment]
    meta: dict[str, Any]

class FasterWhisperASR:
    def __init__(
        self,
        model_size_or_path: str = "large-v3",
        device: str | None = None,
        compute_type: str | None = None,
    ) -> None:
        try:
            from faster_whisper import WhisperModel
        except Exception as e:  # pragma: no cover
            raise RuntimeError(
                "faster-whisper is not installed. Install with: pip install -e '.[asr]'"
            ) from e

        if device is None:
            try:
                import torch

                device = "cuda" if torch.cuda.is_available() else "cpu"
            except Exception:
                device = "cpu"

        if compute_type is None:
            compute_type = "float16" if device == "cuda" else "float32"

        self.model_size_or_path = model_size_or_path


**Planning immutable ASRResult update**

Since ASRResult is frozen, I'll create new instances with modified fields using dataclasses.replace, adding distinct keys for raw and proofread results, and prioritizing proofread as primary when applicable. I'll also extend transcribe with a mode and optional prompt to handle proofread differently, defaulting to transcription when chat mode is unsupported.


**Refining proofread support and prompt building**

I'm formalizing proofread mode to require Voxtral, implementing a dedicated chat prompt builder using mistral_common, and adding flags in the CLI and pipeline for proofread options. I'll detect Voxtral models via ID and handle transcribe versus chat prompt construction accordingly, ensuring proper sampling params and fallback behavior. Next steps include updating pipeline, CLI, and testing two-pass proofread runs with Voxtral.


sed -n '1,120p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/tokenizers/mistral.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast

from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
from vllm.logger import init_logger

from .protocol import TokenizerLike

if TYPE_CHECKING:
    from mistral_common.protocol.instruct.request import (
        ChatCompletionRequest as MistralChatCompletionRequest,
    )
    from mistral_common.tokens.tokenizers.tekken import Tekkenizer
    from transformers import BatchEncoding

    try:
        # Transformers v5
        from transformers.tokenization_mistral_common import MistralCommonBackend
    except ImportError:
        # Transformers v4
        from transformers.tokenization_mistral_common import (
            MistralCommonTokenizer as MistralCommonBackend,
        )

logger = init_logger(__name__)

def maybe_serialize_tool_calls(request: "MistralChatCompletionRequest"):
    # SEE: https://github.com/vllm-project/vllm/pull/9951
    # Credits go to: @gcalmettes
    # NOTE: There is currently a bug in pydantic where attributes
    # declared as iterables are replaced in in the instances by
    # pydantic-core ValidatorIterator instance. In particular, this
    # affects tool_calls defined in ChatCompletionAssistantMessageParam
    # model:
    # see:
    #   - https://github.com/pydantic/pydantic/issues/9467
    # As a result, tool_calls from assistant messages are never
    # deserialized in the request object if the tool_calls iterator is
    # not consumed. This affect messages passed to the MistralTokenizer
    # since no chat template is applied and therefore the tools_calls
    # iterator is not directly consumed.
    # Issue is tracked on Pydantic side, with resolution planned for
    # v2.11 release. In the meantime, the official workaround is to
    # consume the iterator so the tool_calls are correctly deserialized
    # in the OpenAI ChatCompletionAssistantMessageParam object
    # https://github.com/pydantic/pydantic/issues/9467#issuecomment-2442097291 # noqa: E501
    # Official Pydantic Issues:
    #   - https://github.com/pydantic/pydantic/issues/9541
    # TODO: remove when pydantic v2.11 is released
    for i, message in enumerate(request.messages):
        if message.get("role") == "assistant":
            tool_calls_validator = message.get("tool_calls", ().__iter__())
            validated_tool_calls = []
            while True:
                try:
                    tool_call = next(tool_calls_validator)  # type: ignore
                    validated_tool_calls.append(tool_call)
                except StopIteration:
                    break

            request.messages[i]["tool_calls"] = validated_tool_calls

def truncate_tool_call_ids(request: "MistralChatCompletionRequest"):
    """Truncates tool call IDs for Mistral's ID requirements."""
    for i, message in enumerate(request.messages):
        if message.get("role") == "assistant":
            tool_calls = message.get("tool_calls", [])
            for tool_call in tool_calls:
                if len(tool_call["id"]) > 9:
                    logger.warning(
                        "Truncating tool call ID: %s to %s",
                        tool_call["id"],
                        tool_call["id"][-9:],
                    )
                    tool_call["id"] = tool_call["id"][-9:]

            request.messages[i]["tool_calls"] = tool_calls

        elif message.get("role") in {"tool_results", "tool"}:
            if "tool_call_id" in message:
                tool_call_id = message["tool_call_id"]

                if len(tool_call_id) > 9:
                    logger.warning(
                        "Truncating tool_call_id: %s to %s",
                        tool_call_id,
                        tool_call_id[-9:],
                    )
                    tool_call_id = tool_call_id[-9:]
                request.messages[i]["tool_call_id"] = tool_call_id

def _prepare_apply_chat_template_tools_and_messages(
    messages: list["ChatCompletionMessageParam"],
    tools: list[dict[str, Any]] | None = None,
    continue_final_message: bool = False,
    add_generation_prompt: bool = False,
) -> tuple[list["ChatCompletionMessageParam"], list[dict[str, Any]] | None]:
    from mistral_common.protocol.instruct.tool_calls import Function, Tool

    if add_generation_prompt and continue_final_message:
        raise ValueError(
            "Cannot set both `add_generation_prompt` and "
            "`continue_final_message` to True."
        )

    last_message = cast(dict[str, Any], messages[-1])
    # add_generation_prompt is directly handled by the tokenizer but we
    # check if the user is trying to use it with a final assistant message
    # which is probably not what they want.
    # If add_generation_prompt is False, we don't need to check anything.
    if add_generation_prompt and last_message["role"] == "assistant":
        raise ValueError(
            "Cannot set `add_generation_prompt` to True when "
            "the last message is from the assistant. Consider "


**Inspecting vLLM tokenizer wrapper**

I'm checking how the MistralTokenizer wrapper and its `mistral` property relate, focusing on `cached_tokenizer_from_config` to understand the connection and usage of `encode_chat_completion`.


rg -n "def cached_tokenizer_from_config" /home/ubuntu/.local/lib/python3.10/site-packages/vllm/tokenizers -S


Output:
Exit code: 0
Wall time: 0 seconds
Output:
/home/ubuntu/.local/lib/python3.10/site-packages/vllm/tokenizers/registry.py:215:def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):


sed -n '200,260p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/tokenizers/registry.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
        tokenizer_cls_ = tokenizer_cls

    tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs)
    if not tokenizer.is_fast:
        logger.warning(
            "Using a slow tokenizer. This might cause a significant "
            "slowdown. Consider using a fast tokenizer instead."
        )

    return tokenizer  # type: ignore

cached_get_tokenizer = lru_cache(get_tokenizer)

def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
    if model_config.skip_tokenizer_init:
        return None

    return cached_get_tokenizer(
        model_config.tokenizer,
        runner_type=model_config.runner_type,
        tokenizer_mode=model_config.tokenizer_mode,
        revision=model_config.tokenizer_revision,
        trust_remote_code=model_config.trust_remote_code,
        **kwargs,
    )

@deprecated(
    "Renamed to `cached_tokenizer_from_config`. The old name will be removed in v0.14."
)
def init_tokenizer_from_config(model_config: "ModelConfig"):
    return cached_tokenizer_from_config(model_config)


**Clarifying tokenizer structure**

The tokenizer returned by `cached_tokenizer_from_config` likely has nested attributes like `.instruct` or `.mistral`, with `MistralTokenizer` accessed through these properties; I'll check `VoxtralProcessingInfo` to confirm how `get_tokenizer` relates and clarify which tokenizer instance is used in different contexts.


rg -n "class VoxtralProcessingInfo" /home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
181:class VoxtralProcessingInfo(BaseProcessingInfo):


Summary: auto


sed -n '160,220p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/voxtral.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
            # pad if necessary
            try:
                audio = self._audio_processor.pad(audio, self.sampling_rate, False)
            except TypeError:
                audio = self._audio_processor.pad(audio, self.sampling_rate)

            audio_tokens = [self.begin_audio_token_id] + [
                self.audio_token_id
            ] * self.get_num_audio_tokens(len(audio))

            audios_tokens.append(torch.tensor(audio_tokens))
            audios_processed.append(torch.tensor(audio))

        return BatchFeature(
            {
                "input_ids": torch.cat(audios_tokens)[None].expand(len(text), -1),
                "audio_arrays": audios_processed,
            }
        )

class VoxtralProcessingInfo(BaseProcessingInfo):
    def get_tokenizer(self) -> MistralTokenizer:
        tokenizer = cached_tokenizer_from_config(self.ctx.model_config)
        if not isinstance(tokenizer, MistralTokenizer):
            raise ValueError("This model requires `--tokenizer-mode mistral`")

        return tokenizer

    def get_hf_processor(self) -> VoxtralProcessorAdapter:
        return VoxtralProcessorAdapter(self.get_tokenizer())

    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
        return {"audio": 5}  # Performance tends to degrade after 5

    def get_mm_max_tokens_per_item(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
    ) -> Mapping[str, int]:
        return {"audio": self.get_max_audio_tokens()}

    def get_max_audio_tokens(self) -> int:
        return self.ctx.model_config.max_model_len

    def get_max_audio_array_len(self) -> int:
        processor = self.get_hf_processor()
        return self.get_max_audio_tokens() * int(
            processor.sampling_rate // processor.frame_rate
        )

class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
    def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
        return ""

    def get_dummy_mm_data(
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
        mm_options: Mapping[str, BaseDummyOptions] | None = None,


**Inspecting MistralTokenizer class location**

I'm verifying that MistralTokenizer is defined in `vllm/tokenizers/mistral.py` to understand its role and relationship to the `.mistral` attribute.


rg -n "class MistralTokenizer" /home/ubuntu/.local/lib/python3.10/site-packages/vllm/tokenizers/mistral.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
202:class MistralTokenizer(TokenizerLike):


sed -n '200,320p' /home/ubuntu/.local/lib/python3.10/site-packages/vllm/tokenizers/mistral.py