1→# coding=utf-8
     2→# Copyright 2026 The Alibaba Qwen team.
     3→# SPDX-License-Identifier: Apache-2.0
     4→#
     5→# Licensed under the Apache License, Version 2.0 (the "License");
     6→# you may not use this file except in compliance with the License.
     7→# You may obtain a copy of the License at
     8→#
     9→#     http://www.apache.org/licenses/LICENSE-2.0
    10→#
    11→# Unless required by applicable law or agreed to in writing, software
    12→# distributed under the License is distributed on an "AS IS" BASIS,
    13→# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14→# See the License for the specific language governing permissions and
    15→# limitations under the License.
    16→"""
    17→A gradio demo for Qwen3 ASR models.
    18→"""
    19→
    20→import argparse
    21→import base64
    22→import io
    23→import json
    24→import os
    25→from typing import Any, Dict, List, Optional, Tuple, Union
    26→
    27→import gradio as gr
    28→import numpy as np
    29→import torch
    30→from qwen_asr import Qwen3ASRModel
    31→from scipy.io.wavfile import write as wav_write
    32→
    33→
    34→def _title_case_display(s: str) -> str:
    35→    s = (s or "").strip()
    36→    s = s.replace("_", " ")
    37→    return " ".join([w[:1].upper() + w[1:] if w else "" for w in s.split()])
    38→
    39→
    40→def _build_choices_and_map(items: Optional[List[str]]) -> Tuple[List[str], Dict[str, str]]:
    41→    if not items:
    42→        return [], {}
    43→    display = [_title_case_display(x) for x in items]
    44→    mapping = {d: r for d, r in zip(display, items)}
    45→    return display, mapping
    46→
    47→
    48→def _dtype_from_str(s: str) -> torch.dtype:
    49→    s = (s or "").strip().lower()
    50→    if s in ("bf16", "bfloat16"):
    51→        return torch.bfloat16
    52→    if s in ("fp16", "float16", "half"):
    53→        return torch.float16
    54→    if s in ("fp32", "float32"):
    55→        return torch.float32
    56→    raise ValueError(f"Unsupported torch dtype: {s}. Use bfloat16/float16/float32.")
    57→
    58→
    59→def _normalize_audio(wav, eps=1e-12, clip=True):
    60→    x = np.asarray(wav)
    61→
    62→    if np.issubdtype(x.dtype, np.integer):
    63→        info = np.iinfo(x.dtype)
    64→        if info.min < 0:
    65→            y = x.astype(np.float32) / max(abs(info.min), info.max)
    66→        else:
    67→            mid = (info.max + 1) / 2.0
    68→            y = (x.astype(np.float32) - mid) / mid
    69→    elif np.issubdtype(x.dtype, np.floating):
    70→        y = x.astype(np.float32)
    71→        m = np.max(np.abs(y)) if y.size else 0.0
    72→        if m > 1.0 + 1e-6:
    73→            y = y / (m + eps)
    74→    else:
    75→        raise TypeError(f"Unsupported dtype: {x.dtype}")
    76→
    77→    if clip:
    78→        y = np.clip(y, -1.0, 1.0)
    79→
    80→    if y.ndim > 1:
    81→        y = np.mean(y, axis=-1).astype(np.float32)
    82→
    83→    return y
    84→
    85→
    86→def _audio_to_tuple(audio: Any) -> Optional[Tuple[np.ndarray, int]]:
    87→    """
    88→    Accept gradio audio:
    89→      - {"sampling_rate": int, "data": np.ndarray}
    90→      - (sr, np.ndarray)  [some gradio versions]
    91→    Return: (wav_float32_mono, sr)
    92→    """
    93→    if audio is None:
    94→        return None
    95→
    96→    if isinstance(audio, dict) and "sampling_rate" in audio and "data" in audio:
    97→        sr = int(audio["sampling_rate"])
    98→        wav = _normalize_audio(audio["data"])
    99→        return wav, sr
   100→
   101→    if isinstance(audio, tuple) and len(audio) == 2:
   102→        a0, a1 = audio
   103→        if isinstance(a0, int):
   104→            sr = int(a0)
   105→            wav = _normalize_audio(a1)
   106→            return wav, sr
   107→        if isinstance(a1, int):
   108→            wav = _normalize_audio(a0)
   109→            sr = int(a1)
   110→            return wav, sr
   111→
   112→    return None
   113→
   114→
   115→def _parse_audio_any(audio: Any) -> Union[str, Tuple[np.ndarray, int]]:
   116→    if audio is None:
   117→        raise ValueError("Audio is required.")
   118→    at = _audio_to_tuple(audio)
   119→    if at is not None:
   120→        return at
   121→    raise ValueError("Unsupported audio input format.")
   122→
   123→
   124→def build_parser() -> argparse.ArgumentParser:
   125→    parser = argparse.ArgumentParser(
   126→        prog="qwen-asr-demo",
   127→        description=(
   128→            "Launch a Gradio demo for Qwen3 ASR models (Transformers / vLLM).\n\n"
   129→            "Examples:\n"
   130→            "  qwen-asr-demo --asr-checkpoint Qwen/Qwen3-ASR-1.7B\n"
   131→            "  qwen-asr-demo --asr-checkpoint Qwen/Qwen3-ASR-1.7B --aligner-checkpoint Qwen/Qwen3-ForcedAligner-0.6B\n"
   132→            "  qwen-asr-demo --backend vllm --cuda-visible-devices 0\n"
   133→            "  qwen-asr-demo --backend transformers --backend-kwargs '{\"device_map\":\"cuda:0\",\"dtype\":\"bfloat16\",\"attn_implementation\":\"flash_attention_2\"}'\n"
   134→            "  qwen-asr-demo --backend vllm --backend-kwargs '{\"gpu_memory_utilization\":0.85}'\n"
   135→        ),
   136→        formatter_class=argparse.RawTextHelpFormatter,
   137→        add_help=True,
   138→    )
   139→
   140→    parser.add_argument("--asr-checkpoint", required=True, help="Qwen3-ASR model checkpoint path or HF repo id.")
   141→    parser.add_argument(
   142→        "--aligner-checkpoint",
   143→        default=None,
   144→        help="Qwen3-ForcedAligner checkpoint path or HF repo id (optional; enables timestamps when provided).",
   145→    )
   146→
   147→    parser.add_argument(
   148→        "--backend",
   149→        default="transformers",
   150→        choices=["transformers", "vllm"],
   151→        help="Backend for ASR model loading (default: transformers).",
   152→    )
   153→
   154→    parser.add_argument(
   155→        "--cuda-visible-devices",
   156→        default="0",
   157→        help=(
   158→            "Set CUDA_VISIBLE_DEVICES for the demo process (default: 0). "
   159→            "Use e.g. '0' or '1'"
   160→        ),
   161→    )
   162→
   163→    parser.add_argument(
   164→        "--backend-kwargs",
   165→        default=None,
   166→        help=(
   167→            "JSON dict for backend-specific kwargs excluding checkpoints.\n"
   168→            "Examples:\n"
   169→            "  transformers: '{\"device_map\":\"cuda:0\",\"dtype\":\"bfloat16\",\"attn_implementation\":\"flash_attention_2\",\"max_inference_batch_size\":32}'\n"
   170→            "  vllm        : '{\"gpu_memory_utilization\":0.8,\"max_inference_batch_size\":32}'\n"
   171→        ),
   172→    )
   173→    parser.add_argument(
   174→        "--aligner-kwargs",
   175→        default=None,
   176→        help=(
   177→            "JSON dict for forced aligner kwargs (only used when --aligner-checkpoint is set).\n"
   178→            "Example: '{\"dtype\":\"bfloat16\",\"device_map\":\"cuda:0\"}'\n"
   179→        ),
   180→    )
   181→
   182→    # Gradio server args
   183→    parser.add_argument("--ip", default="0.0.0.0", help="Server bind IP for Gradio (default: 0.0.0.0).")
   184→    parser.add_argument("--port", type=int, default=8000, help="Server port for Gradio (default: 8000).")
   185→    parser.add_argument(
   186→        "--share/--no-share",
   187→        dest="share",
   188→        default=False,
   189→        action=argparse.BooleanOptionalAction,
   190→        help="Whether to create a public Gradio link (default: disabled).",
   191→    )
   192→    parser.add_argument("--concurrency", type=int, default=16, help="Gradio queue concurrency (default: 16).")
   193→
   194→    # HTTPS args
   195→    parser.add_argument("--ssl-certfile", default=None, help="Path to SSL certificate file for HTTPS (optional).")
   196→    parser.add_argument("--ssl-keyfile", default=None, help="Path to SSL key file for HTTPS (optional).")
   197→    parser.add_argument(
   198→        "--ssl-verify/--no-ssl-verify",
   199→        dest="ssl_verify",
   200→        default=True,
   201→        action=argparse.BooleanOptionalAction,
   202→        help="Whether to verify SSL certificate (default: enabled).",
   203→    )
   204→
   205→    return parser
   206→
   207→
   208→def _parse_json_dict(s: Optional[str], *, name: str) -> Dict[str, Any]:
   209→    if s is None or not str(s).strip():
   210→        return {}
   211→    try:
   212→        obj = json.loads(s)
   213→    except Exception as e:
   214→        raise ValueError(f"Invalid JSON for {name}: {e}")
   215→    if not isinstance(obj, dict):
   216→        raise ValueError(f"{name} must be a JSON object (dict).")
   217→    return obj
   218→
   219→
   220→def _apply_cuda_visible_devices(cuda_visible_devices: str) -> None:
   221→    v = (cuda_visible_devices or "").strip()
   222→    if not v:
   223→        return
   224→    os.environ["CUDA_VISIBLE_DEVICES"] = v
   225→
   226→
   227→def _default_backend_kwargs(backend: str) -> Dict[str, Any]:
   228→    if backend == "transformers":
   229→        return dict(
   230→            dtype=torch.bfloat16,
   231→            device_map="cuda:0",
   232→            max_inference_batch_size=4,
   233→            max_new_tokens=512,
   234→        )
   235→    else:
   236→        return dict(
   237→            gpu_memory_utilization=0.8,
   238→            max_inference_batch_size=4,
   239→            max_new_tokens=4096,
   240→        )
   241→
   242→
   243→def _default_aligner_kwargs() -> Dict[str, Any]:
   244→    return dict(
   245→        dtype=torch.bfloat16,
   246→        device_map="cuda:0",
   247→    )
   248→
   249→
   250→def _merge_dicts(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
   251→    out = dict(base)
   252→    out.update(override)
   253→    return out
   254→
   255→
   256→def _coerce_special_types(d: Dict[str, Any]) -> Dict[str, Any]:
   257→    out: Dict[str, Any] = {}
   258→    for k, v in d.items():
   259→        if k == "dtype" and isinstance(v, str):
   260→            out[k] = _dtype_from_str(v)
   261→        else:
   262→            out[k] = v
   263→    return out
   264→
   265→
   266→def _make_timestamp_html(audio_upload: Any, timestamps: Any) -> str:
   267→    """
   268→    Build HTML with per-token audio slices, using base64 data URLs.
   269→    Expect timestamps as list[dict] with keys: text, start_time, end_time (ms).
   270→    """
   271→    at = _audio_to_tuple(audio_upload)
   272→    if at is None:
   273→        raise ValueError("Audio input is required for visualization.")
   274→    audio, sr = at
   275→
   276→    if not timestamps:
   277→        return "
No timestamps to visualize.
"
   278→    if not isinstance(timestamps, list):
   279→        raise ValueError("Timestamps must be a list (JSON array).")
   280→
   281→    html_content = """
   282→
   295→    """
   296→
   297→    html_content += """
   298→
   299→        Timestamps Visualization (时间戳可视化结果）
   300→

   301→    """
   302→
   303→    for item in timestamps:
   304→        if not isinstance(item, dict):
   305→            continue
   306→        word = str(item.get("text", "") or "")
   307→        start = item.get("start_time", None)
   308→        end = item.get("end_time", None)
   309→        if start is None or end is None:
   310→            continue
   311→
   312→        start = float(start)
   313→        end = float(end)
   314→        if end <= start:
   315→            continue
   316→
   317→        start_sample = max(0, int(start * sr))
   318→        end_sample = min(len(audio), int(end * sr))
   319→        if end_sample <= start_sample:
   320→            continue
   321→
   322→        seg = audio[start_sample:end_sample]
   323→        seg_i16 = (np.clip(seg, -1.0, 1.0) * 32767.0).astype(np.int16)
   324→
   325→        mem = io.BytesIO()
   326→        wav_write(mem, sr, seg_i16)
   327→        mem.seek(0)
   328→        b64 = base64.b64encode(mem.read()).decode("utf-8")
   329→        audio_src = f"data:audio/wav;base64,{b64}"
   330→
   331→        html_content += f"""
   332→

   333→
{word}

   334→
{start} - {end} s

   335→

   336→                <audio controls preload="none" src="{audio_src}"></audio>
   337→

   338→

   339→        """
   340→
   341→    html_content += "
"
   342→    return html_content
   343→
   344→
   345→def build_demo(
   346→    asr: Qwen3ASRModel,
   347→    asr_ckpt: str,
   348→    backend: str,
   349→    aligner_ckpt: Optional[str] = None,
   350→) -> gr.Blocks:
   351→    supported_langs_raw = asr.get_supported_languages()
   352→    lang_choices_disp, lang_map = _build_choices_and_map([x for x in supported_langs_raw])
   353→    lang_choices = ["Auto"] + lang_choices_disp
   354→
   355→    has_aligner = bool(aligner_ckpt)
   356→
   357→    theme = gr.themes.Soft(
   358→        font=[gr.themes.GoogleFont("Source Sans Pro"), "Arial", "sans-serif"],
   359→    )
   360→    css = ".gradio-container {max-width: none !important;}"
   361→
   362→    with gr.Blocks(theme=theme, css=css) as demo:
   363→        gr.Markdown(
   364→            f"""
   365→# Qwen3 ASR Demo
   366→**Backend:** `{backend}`
   367→**ASR Checkpoint:** `{asr_ckpt}`
   368→**Forced Aligner:** `{aligner_ckpt if aligner_ckpt else "(none)"}`
   369→"""
   370→        )
   371→
   372→        with gr.Row():
   373→            with gr.Column(scale=2):
   374→                audio_in = gr.Audio(label="Audio Input (上传音频)", type="numpy")
   375→                lang_in = gr.Dropdown(
   376→                    label="Language (语种)",
   377→                    choices=lang_choices,
   378→                    value="Auto",
   379→                    interactive=True,
   380→                )
   381→                if has_aligner:
   382→                    ts_in = gr.Checkbox(
   383→                        label="Return Timestamps (是否返回时间戳)",
   384→                        value=True,
   385→                    )
   386→                else:
   387→                    ts_in = gr.State(False)
   388→
   389→                btn = gr.Button("Transcribe (识别)", variant="primary")
   390→
   391→            with gr.Column(scale=2):
   392→                out_lang = gr.Textbox(label="Detected Language", lines=1)
   393→                out_text = gr.Textbox(label="Result Text", lines=12)
   394→
   395→            if has_aligner:
   396→                with gr.Column(scale=3):
   397→                    out_ts = gr.JSON(label="Timestamps（时间戳结果）")
   398→                    viz_btn = gr.Button("Visualize Timestamps (可视化时间戳)", variant="secondary")
   399→            else:
   400→                with gr.Column(scale=3):
   401→                    out_ts = gr.State(None)
   402→                    viz_btn = gr.State(None)
   403→
   404→        # Put the visualization panel below the three columns
   405→        if has_aligner:
   406→            with gr.Row():
   407→                out_ts_html = gr.HTML(label="Timestamps Visualization (时间戳可视化结果)")
   408→        else:
   409→            out_ts_html = gr.State("")
   410→
   411→        def run(audio_upload: Any, lang_disp: str, return_ts: bool):
   412→            audio_obj = _parse_audio_any(audio_upload)
   413→
   414→            language = None
   415→            if lang_disp and lang_disp != "Auto":
   416→                language = lang_map.get(lang_disp, lang_disp)
   417→
   418→            return_ts = bool(return_ts) and has_aligner
   419→
   420→            results = asr.transcribe(
   421→                audio=audio_obj,
   422→                language=language,
   423→                return_time_stamps=return_ts,
   424→            )
   425→            if not isinstance(results, list) or len(results) != 1:
   426→                raise RuntimeError(
   427→                    f"Unexpected result size: {type(results)} "
   428→                    f"len={len(results) if isinstance(results, list) else 'N/A'}"
   429→                )
   430→
   431→            r = results[0]
   432→
   433→            if has_aligner:
   434→                ts_payload = None
   435→                if return_ts:
   436→                    ts_payload = [
   437→                        dict(
   438→                            text=getattr(t, "text", None),
   439→                            start_time=getattr(t, "start_time", None),
   440→                            end_time=getattr(t, "end_time", None),
   441→                        )
   442→                        for t in (getattr(r, "time_stamps", None) or [])
   443→                    ]
   444→                return (
   445→                    getattr(r, "language", "") or "",
   446→                    getattr(r, "text", "") or "",
   447→                    gr.update(value=ts_payload) if return_ts else gr.update(value=None),
   448→                    gr.update(value=""),  # clear html on each transcribe
   449→                )
   450→            else:
   451→                return (
   452→                    getattr(r, "language", "") or "",
   453→                    getattr(r, "text", "") or "",
   454→                )
   455→
   456→        def visualize(audio_upload: Any, timestamps_json: Any):
   457→            return _make_timestamp_html(audio_upload, timestamps_json)
   458→
   459→        if has_aligner:
   460→            btn.click(
   461→                run,
   462→                inputs=[audio_in, lang_in, ts_in],
   463→                outputs=[out_lang, out_text, out_ts, out_ts_html],
   464→            )
   465→            viz_btn.click(
   466→                visualize,
   467→                inputs=[audio_in, out_ts],
   468→                outputs=[out_ts_html],
   469→            )
   470→        else:
   471→            btn.click(
   472→                run,
   473→                inputs=[audio_in, lang_in, ts_in],
   474→                outputs=[out_lang, out_text],
   475→            )
   476→
   477→    return demo
   478→
   479→
   480→def main(argv=None) -> int:
   481→    parser = build_parser()
   482→    args = parser.parse_args(argv)
   483→
   484→    _apply_cuda_visible_devices(args.cuda_visible_devices)
   485→
   486→    backend = args.backend
   487→    asr_ckpt = args.asr_checkpoint
   488→    aligner_ckpt = args.aligner_checkpoint
   489→
   490→    user_backend_kwargs = _parse_json_dict(args.backend_kwargs, name="--backend-kwargs")
   491→    user_aligner_kwargs = _parse_json_dict(args.aligner_kwargs, name="--aligner-kwargs")
   492→
   493→    backend_kwargs = _merge_dicts(_default_backend_kwargs(backend), user_backend_kwargs)
   494→    backend_kwargs = _coerce_special_types(backend_kwargs)
   495→
   496→    forced_aligner = None
   497→    forced_aligner_kwargs = None
   498→    if aligner_ckpt:
   499→        forced_aligner = aligner_ckpt
   500→        aligner_kwargs = _merge_dicts(_default_aligner_kwargs(), user_aligner_kwargs)
   501→        forced_aligner_kwargs = _coerce_special_types(aligner_kwargs)
   502→
   503→    if backend == "transformers":
   504→        asr = Qwen3ASRModel.from_pretrained(
   505→            asr_ckpt,
   506→            forced_aligner=forced_aligner,
   507→            forced_aligner_kwargs=forced_aligner_kwargs,
   508→            **backend_kwargs,
   509→        )
   510→    else:
   511→        asr = Qwen3ASRModel.LLM(
   512→            model=asr_ckpt,
   513→            forced_aligner=forced_aligner,
   514→            forced_aligner_kwargs=forced_aligner_kwargs,
   515→            **backend_kwargs,
   516→        )
   517→
   518→    demo = build_demo(asr, asr_ckpt, backend, aligner_ckpt=aligner_ckpt)
   519→
   520→    launch_kwargs: Dict[str, Any] = dict(
   521→        server_name=args.ip,
   522→        server_port=args.port,
   523→        share=args.share,
   524→        ssl_verify=True if args.ssl_verify else False,
   525→    )
   526→    if args.ssl_certfile is not None:
   527→        launch_kwargs["ssl_certfile"] = args.ssl_certfile
   528→    if args.ssl_keyfile is not None:
   529→        launch_kwargs["ssl_keyfile"] = args.ssl_keyfile
   530→
   531→    demo.queue(default_concurrency_limit=int(args.concurrency)).launch(**launch_kwargs)
   532→    return 0
   533→
   534→
   535→if __name__ == "__main__":
   536→    raise SystemExit(main())
   537→


1→# Qwen3-ASR
     2→
     3→

     4→
     5→

     6→
     7→

     8→
     9→

    10→&nbsp&nbsp🤗 Hugging Face&nbsp&nbsp | &nbsp&nbsp🤖 ModelScope&nbsp&nbsp | &nbsp&nbsp📑 Blog&nbsp&nbsp | &nbsp&nbsp📑 Paper&nbsp&nbsp
    11→

    12→🖥 Hugging Face Demo&nbsp&nbsp | &nbsp&nbsp 🖥 ModelScope Demo&nbsp&nbsp | &nbsp&nbsp💬 WeChat (微信)&nbsp&nbsp | &nbsp&nbsp🫨 Discord&nbsp&nbsp | &nbsp&nbsp📑 API
    13→
    14→

    15→
    16→We release **Qwen3-ASR**, a family that includes two powerful all-in-one speech recognition models that support language identification and ASR for 52 languages and dialects, as well as a novel non-autoregressive speech forced-alignment model that can align text–speech pairs in 11 languages.
    17→
    18→
    19→## News
    20→* 2026.1.29: 🎉🎉🎉 We have released the [Qwen3-ASR](https://huggingface.co/collections/Qwen/qwen3-asr) series (0.6B/1.7B) and the Qwen3-ForcedAligner-0.6B model. Please check out our [blog](https://qwen.ai/blog?id=qwen3asr)!
    21→
    22→
    23→## Contents
    24→
    25→- [Overview](#overview)
    26→  - [Introduction](#introduction)
    27→  - [Model Architecture](#model-architecture)
    28→  - [Released Models Description and Download](#released-models-description-and-download)
    29→- [Quickstart](#quickstart)
    30→  - [Environment Setup](#environment-setup)
    31→  - [Python Package Usage](#python-package-usage)
    32→    - [Quick Inference](#quick-inference)
    33→    - [vLLM Backend](#vllm-backend)
    34→    - [Streaming Inference](#streaming-inference)
    35→    - [ForcedAligner Usage](#forcedaligner-usage)
    36→  - [DashScope API Usage](#dashscope-api-usage)
    37→- [Launch Local Web UI Demo](#launch-local-web-ui-demo)
    38→  - [Gradio Demo](#gradio-demo)
    39→  - [Streaming Demo](#streaming-demo)
    40→- [Deployment with vLLM](#deployment-with-vllm)
    41→- [Fine Tuning](#fine-tuning)
    42→- [Docker](#docker)
    43→- [Evaluation](#evaluation)
    44→- [Citation](#citation)
    45→
    46→
    47→## Overview
    48→
    49→### Introduction
    50→
    51→

    52→
    53→

    54→
    55→The Qwen3-ASR family includes Qwen3-ASR-1.7B and Qwen3-ASR-0.6B, which support language identification and ASR for 52 languages and dialects. Both leverage large-scale speech training data and the strong audio understanding capability of their foundation model, Qwen3-Omni. Experiments show that the 1.7B version achieves state-of-the-art performance among open-source ASR models and is competitive with the strongest proprietary commercial APIs. Here are the main features:
    56→
    57→* **All-in-one**: Qwen3-ASR-1.7B and Qwen3-ASR-0.6B support language identification and speech recognition for 30 languages and 22 Chinese dialects, so as to English accents from multiple countries and regions.
    58→
    59→* **Excellent and Fast**: The Qwen3-ASR family ASR models maintains high-quality and robust recognition under complex acoustic environments and challenging text patterns. Qwen3-ASR-1.7B achieves strong performance on both open-sourced and internal benchmarks. While the 0.6B version achieves accuracy-efficient trade-off, it reaches 2000 times throughput at a concurrency of 128. They both achieve streaming / offline unified inference with single model and support transcribe long audio.
    60→
    61→* **Novel and strong forced alignment Solution**: We introduce Qwen3-ForcedAligner-0.6B, which supports timestamp prediction for arbitrary units within up to 5 minutes of speech in 11 languages. Evaluations show its timestamp accuracy surpasses E2E based forced-alignment models.
    62→
    63→* **Comprehensive inference toolkit**: In addition to open-sourcing the architectures and weights of the Qwen3-ASR series, we also release a powerful, full-featured inference framework that supports vLLM-based batch inference, asynchronous serving, streaming inference, timestamp prediction, and more.
    64→
    65→### Model Architecture
    66→
    67→

    68→
    69→

    70→
    71→
    72→### Released Models Description and Download
    73→
    74→Below is an introduction and download information for the Qwen3-ASR models. Please select and download the model that fits your needs.
    75→
    76→| Model | Supported Languages | Supported Dialects | Inference Mode | Audio Types |
    77→|---|---|---|---|---|
    78→| Qwen3-ASR-1.7B & Qwen3-ASR-0.6B | Chinese (zh), English (en), Cantonese (yue), Arabic (ar), German (de), French (fr), Spanish (es), Portuguese (pt), Indonesian (id), Italian (it), Korean (ko), Russian (ru), Thai (th), Vietnamese (vi), Japanese (ja), Turkish (tr), Hindi (hi), Malay (ms), Dutch (nl), Swedish (sv), Danish (da), Finnish (fi), Polish (pl), Czech (cs), Filipino (fil), Persian (fa), Greek (el), Hungarian (hu), Macedonian (mk), Romanian (ro) | Anhui, Dongbei, Fujian, Gansu, Guizhou, Hebei, Henan, Hubei, Hunan, Jiangxi, Ningxia, Shandong, Shaanxi, Shanxi, Sichuan, Tianjin, Yunnan, Zhejiang, Cantonese (Hong Kong accent), Cantonese (Guangdong accent), Wu language, Minnan language. | Offline / Streaming | Speech, Singing Voice, Songs with BGM |
    79→| Qwen3-ForcedAligner-0.6B | Chinese, English, Cantonese, French, German, Italian, Japanese, Korean, Portuguese, Russian, Spanish | -- | NAR | Speech |
    80→
    81→During model loading in the `qwen-asr` package or vLLM, model weights will be downloaded automatically based on the model name. However, if your runtime environment does not allow downloading weights during execution, you can use the following commands to manually download the model weights to a local directory:
    82→
    83→```bash
    84→# Download through ModelScope (recommended for users in Mainland China)
    85→pip install -U modelscope
    86→modelscope download --model Qwen/Qwen3-ASR-1.7B  --local_dir ./Qwen3-ASR-1.7B
    87→modelscope download --model Qwen/Qwen3-ASR-0.6B --local_dir ./Qwen3-ASR-0.6B
    88→modelscope download --model Qwen/Qwen3-ForcedAligner-0.6B --local_dir ./Qwen3-ForcedAligner-0.6B
    89→# Download through Hugging Face
    90→pip install -U "huggingface_hub[cli]"
    91→huggingface-cli download Qwen/Qwen3-ASR-1.7B --local-dir ./Qwen3-ASR-1.7B
    92→huggingface-cli download Qwen/Qwen3-ASR-0.6B --local-dir ./Qwen3-ASR-0.6B
    93→huggingface-cli download Qwen/Qwen3-ForcedAligner-0.6B --local-dir ./Qwen3-ForcedAligner-0.6B
    94→```
    95→
    96→
    97→## Quickstart
    98→
    99→### Environment Setup
   100→
   101→The easiest way to use Qwen3-ASR is to install the `qwen-asr` Python package from PyPI. This will pull in the required runtime dependencies and allow you to load any released Qwen3-ASR model. If you’d like to simplify environment setup further, you can also use our official [Docker image](#docker). The `qwen-asr` package provides two backends: the transformers backend and the vLLM backend. For usage instructions for different backends, please refer to [Python Package Usage](#python-package-usage). We recommend using a **fresh, isolated environment** to avoid dependency conflicts with existing packages. You can create a clean Python 3.12 environment like this:
   102→
   103→```bash
   104→conda create -n qwen3-asr python=3.12 -y
   105→conda activate qwen3-asr
   106→```
   107→
   108→Run the following command to get the minimal installation with transformers-backend support:
   109→
   110→```bash
   111→pip install -U qwen-asr
   112→```
   113→
   114→To enable the vLLM backend for faster inference and streaming support, run:
   115→
   116→```bash
   117→pip install -U qwen-asr[vllm]
   118→```
   119→
   120→If you want to develop or modify the code locally, install from source in editable mode:
   121→
   122→```bash
   123→git clone https://github.com/QwenLM/Qwen3-ASR.git
   124→cd Qwen3-ASR
   125→pip install -e .
   126→# support vLLM backend
   127→# pip install -e ".[vllm]"
   128→```
   129→
   130→Additionally, we recommend using FlashAttention 2 to reduce GPU memory usage and accelerate inference speed, especially for long inputs and large batch sizes.
   131→
   132→```bash
   133→pip install -U flash-attn --no-build-isolation
   134→```
   135→
   136→If your machine has less than 96GB of RAM and lots of CPU cores, run:
   137→
   138→```bash
   139→MAX_JOBS=4 pip install -U flash-attn --no-build-isolation
   140→```
   141→
   142→Also, you should have hardware that is compatible with FlashAttention 2. Read more about it in the official documentation of the [FlashAttention repository](https://github.com/Dao-AILab/flash-attention). FlashAttention 2 can only be used when a model is loaded in `torch.float16` or `torch.bfloat16`.
   143→
   144→### Python Package Usage
   145→
   146→#### Quick Inference
   147→
   148→The `qwen-asr` package provides two backends: **transformers backend** and **vLLM backend**. You can pass audio inputs as a local path, a URL, base64 data, or a `(np.ndarray, sr)` tuple, and run batch inference. To quickly try Qwen3-ASR, you can use `Qwen3ASRModel.from_pretrained(...)` for the transformers backend with the following code:
   149→
   150→```python
   151→import torch
   152→from qwen_asr import Qwen3ASRModel
   153→
   154→model = Qwen3ASRModel.from_pretrained(
   155→    "Qwen/Qwen3-ASR-1.7B",
   156→    dtype=torch.bfloat16,
   157→    device_map="cuda:0",
   158→    # attn_implementation="flash_attention_2",
   159→    max_inference_batch_size=32, # Batch size limit for inference. -1 means unlimited. Smaller values can help avoid OOM.
   160→    max_new_tokens=256, # Maximum number of tokens to generate. Set a larger value for long audio input.
   161→)
   162→
   163→results = model.transcribe(
   164→    audio="https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav",
   165→    language=None, # set "English" to force the language
   166→)
   167→
   168→print(results[0].language)
   169→print(results[0].text)
   170→```
   171→
   172→If you want to return timestamps, pass `forced_aligner` and its init kwargs. Here is an example of batch inference with timestamps output:
   173→
   174→```python
   175→import torch
   176→from qwen_asr import Qwen3ASRModel
   177→
   178→model = Qwen3ASRModel.from_pretrained(
   179→    "Qwen/Qwen3-ASR-1.7B",
   180→    dtype=torch.bfloat16,
   181→    device_map="cuda:0",
   182→    # attn_implementation="flash_attention_2",
   183→    max_inference_batch_size=32, # Batch size limit for inference. -1 means unlimited. Smaller values can help avoid OOM.
   184→    max_new_tokens=256, # Maximum number of tokens to generate. Set a larger value for long audio input.
   185→    forced_aligner="Qwen/Qwen3-ForcedAligner-0.6B",
   186→    forced_aligner_kwargs=dict(
   187→        dtype=torch.bfloat16,
   188→        device_map="cuda:0",
   189→        # attn_implementation="flash_attention_2",
   190→    ),
   191→)
   192→
   193→results = model.transcribe(
   194→    audio=[
   195→      "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_zh.wav",
   196→      "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav",
   197→    ],
   198→    language=["Chinese", "English"], # can also be set to None for automatic language detection
   199→    return_time_stamps=True,
   200→)


     1→[build-system]
     2→requires = ["setuptools>=68", "wheel"]
     3→build-backend = "setuptools.build_meta"
     4→
     5→[project]
     6→name = "qwen-asr"
     7→version = "0.0.6"
     8→description = "Qwen-ASR python package"
     9→readme = "README.md"
    10→requires-python = ">=3.9"
    11→classifiers = [
    12→  "Programming Language :: Python :: 3",
    13→  "Programming Language :: Python :: 3.9",
    14→  "Programming Language :: Python :: 3.10",
    15→  "Programming Language :: Python :: 3.11",
    16→  "Programming Language :: Python :: 3.12",
    17→  "Programming Language :: Python :: 3.13",
    18→]
    19→license = { text = "Apache-2.0" }
    20→authors = [{ name = "Alibaba Qwen Team" }]
    21→
    22→dependencies = [
    23→  "transformers==4.57.6",
    24→  "nagisa==0.2.11",
    25→  "soynlp==0.0.493",
    26→  "accelerate==1.12.0",
    27→  "qwen-omni-utils",
    28→  "librosa",
    29→  "soundfile",
    30→  "sox",
    31→  "gradio",
    32→  "flask",
    33→  "pytz",
    34→]
    35→
    36→[project.optional-dependencies]
    37→vllm = [
    38→  "vllm==0.14.0",
    39→]
    40→
    41→[project.urls]
    42→Homepage = "https://github.com/Qwen/Qwen3-ASR"
    43→Repository = "https://github.com/Qwen/Qwen3-ASR"
    44→
    45→[project.scripts]
    46→qwen-asr-demo = "qwen_asr.cli.demo:main"
    47→qwen-asr-demo-streaming = "qwen_asr.cli.demo_streaming:main"
    48→qwen-asr-serve = "qwen_asr.cli.serve:main"
    49→
    50→[tool.setuptools]
    51→packages = { find = { where = ["."] , include = ["qwen_asr*"] } }
    52→include-package-data = true
    53→
    54→[tool.setuptools.package-data]
    55→qwen_tts = ["py.typed", "**/*.dict"]


     1→#!/usr/bin/env bash
     2→# Benchmark series: BS sweep + torch.compile + profiling
     3→set -euo pipefail
     4→
     5→export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
     6→export TORCH_NCCL_ASYNC_ERROR_HANDLING=3
     7→export OMP_NUM_THREADS=1
     8→export MKL_NUM_THREADS=1
     9→export OPENBLAS_NUM_THREADS=1
    10→export NUMEXPR_NUM_THREADS=1
    11→export NCCL_LAUNCH_MODE=PARALLEL
    12→export TORCHINDUCTOR_CACHE_DIR="/tmp/torch_inductor_cache"
    13→export PYTHONPATH="/root/data/Qwen3-ASR-official:${PYTHONPATH:-}"
    14→
    15→MODEL_PATH="/root/data/qwen3_asr_weights"
    16→TRAIN_FILE="/root/gemini-asr/lf_asr/artifacts/phase2/train.parquet"
    17→OUTPUT_DIR="/root/data/qwen3-asr-phase2-bench"
    18→LOG_DIR="/root/data/qwen3-asr-phase2-bench/logs"
    19→STEPS=100
    20→
    21→mkdir -p "$OUTPUT_DIR" "$LOG_DIR"
    22→
    23→run_experiment() {
    24→    local EXP_ID="$1"
    25→    local BS="$2"
    26→    local COMPILE_FLAG="$3"
    27→    local EXTRA_FLAGS="${4:-}"
    28→
    29→    echo ""
    30→    echo "========================================"
    31→    echo "  Experiment: $EXP_ID"
    32→    echo "  BS=$BS, compile=$COMPILE_FLAG"
    33→    echo "  Extra: $EXTRA_FLAGS"
    34→    echo "========================================"
    35→
    36→    # Clean up from previous run
    37→    rm -rf "$OUTPUT_DIR/checkpoint-*" 2>/dev/null || true
    38→
    39→    # GPU snapshot before
    40→    nvidia-smi --query-gpu=index,memory.used,utilization.gpu,power.draw --format=csv,noheader > "$LOG_DIR/${EXP_ID}_gpu_pre.csv"
    41→
    42→    # Background GPU monitor (sample every 5s)
    43→    (while true; do
    44→        nvidia-smi --query-gpu=index,memory.used,utilization.gpu,power.draw --format=csv,noheader
    45→        sleep 5
    46→    done) > "$LOG_DIR/${EXP_ID}_gpu_trace.csv" 2>/dev/null &
    47→    GPU_MON_PID=$!
    48→
    49→    START_TIME=$(date +%s)
    50→
    51→    torchrun --standalone --nproc_per_node=8 \
    52→        finetuning/qwen3_asr_sft_phase2.py \
    53→        --model_path "${MODEL_PATH}" \
    54→        --train_file "${TRAIN_FILE}" \
    55→        --output_dir "${OUTPUT_DIR}" \
    56→        --batch_size "${BS}" \
    57→        --grad_acc 1 \
    58→        --lr 2e-5 \
    59→        --max_steps "${STEPS}" \
    60→        --log_steps 10 \
    61→        --save_steps 999999 \
    62→        --eval_steps 999999 \
    63→        --num_workers 8 \
    64→        --prefetch_factor 4 \
    65→        --pin_memory 1 \
    66→        --persistent_workers 1 \
    67→        --max_open_tars 32 \
    68→        --gradient_checkpointing 0 \
    69→        --ddp_static_graph 1 \
    70→        --language_tag_mode auto \
    71→        --max_duration_s 30.0 \
    72→        --seed 42 \
    73→        ${COMPILE_FLAG} \
    74→        ${EXTRA_FLAGS} \
    75→        2>&1 | tee "$LOG_DIR/${EXP_ID}.log"
    76→
    77→    END_TIME=$(date +%s)
    78→    ELAPSED=$((END_TIME - START_TIME))
    79→
    80→    # Stop GPU monitor
    81→    kill $GPU_MON_PID 2>/dev/null || true
    82→
    83→    # GPU snapshot after
    84→    nvidia-smi --query-gpu=index,memory.used,utilization.gpu,power.draw --format=csv,noheader > "$LOG_DIR/${EXP_ID}_gpu_post.csv"
    85→
    86→    # Extract metrics
    87→    echo ""
    88→    echo "--- $EXP_ID Results ---"
    89→    echo "Wall time: ${ELAPSED}s"
    90→    echo "Steps: $STEPS"
    91→    echo "Avg step time: $(echo "scale=3; $ELAPSED / $STEPS" | bc)s"
    92→
    93→    # Extract peak memory from GPU trace
    94→    if [ -f "$LOG_DIR/${EXP_ID}_gpu_trace.csv" ]; then
    95→        MAX_MEM=$(awk -F', ' '{print $2}' "$LOG_DIR/${EXP_ID}_gpu_trace.csv" | sed 's/ MiB//' | sort -n | tail -1)
    96→        echo "Peak GPU memory: ${MAX_MEM} MiB"
    97→    fi
    98→
    99→    # Extract loss from log
   100→    grep -oP "'loss': [\d.]+" "$LOG_DIR/${EXP_ID}.log" | tail -1
   101→    grep -oP "'train_samples_per_second': [\d.]+" "$LOG_DIR/${EXP_ID}.log" | tail -1
   102→    grep -oP "'train_steps_per_second': [\d.]+" "$LOG_DIR/${EXP_ID}.log" | tail -1
   103→    echo "========================="
   104→}
   105→
   106→echo "Starting benchmark series at $(date)"
   107→
   108→# E1: BS=16, no compile (baseline)
   109→run_experiment "E1_bs16_nocompile" 16 ""
   110→
   111→# E2: BS=32, no compile
   112→run_experiment "E2_bs32_nocompile" 32 ""
   113→
   114→# E3: BS=16, torch.compile reduce-overhead
   115→run_experiment "E3_bs16_compile_ro" 16 "--torch_compile 1 --compile_mode reduce-overhead"
   116→
   117→# E4: BS=32, torch.compile reduce-overhead
   118→run_experiment "E4_bs32_compile_ro" 32 "--torch_compile 1 --compile_mode reduce-overhead"
   119→
   120→# E5: BS=64, no compile (test memory limit)
   121→run_experiment "E5_bs64_nocompile" 64 "" || echo "E5 FAILED (likely OOM)"
   122→
   123→echo ""
   124→echo "All benchmarks complete at $(date)"
   125→echo "Logs in: $LOG_DIR"
   126→


Bash

find /home/ubuntu/training/qwen3-asr-1.7b-phase2-sft -name "*.py" -path "*/inference/*" -type f


     1→# coding=utf-8
     2→# Copyright 2026 The Alibaba Qwen team.
     3→# SPDX-License-Identifier: Apache-2.0
     4→#
     5→# Licensed under the Apache License, Version 2.0 (the "License");
     6→# you may not use this file except in compliance with the License.
     7→# You may obtain a copy of the License at
     8→#
     9→#     http://www.apache.org/licenses/LICENSE-2.0
    10→#
    11→# Unless required by applicable law or agreed to in writing, software
    12→# distributed under the License is distributed on an "AS IS" BASIS,
    13→# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14→# See the License for the specific language governing permissions and
    15→# limitations under the License.
    16→import os
    17→import unicodedata
    18→from dataclasses import dataclass
    19→from typing import Any, Dict, List, Optional, Union
    20→
    21→import nagisa
    22→import torch
    23→from qwen_asr.core.transformers_backend import (
    24→    Qwen3ASRConfig,
    25→    Qwen3ASRForConditionalGeneration,
    26→    Qwen3ASRProcessor,
    27→)
    28→from transformers import AutoConfig, AutoModel, AutoProcessor
    29→
    30→from .utils import (
    31→    AudioLike,
    32→    ensure_list,
    33→    normalize_audios,
    34→)
    35→
    36→
    37→class Qwen3ForceAlignProcessor():
    38→    def __init__(self):
    39→        ko_dict_path = os.path.join(os.path.dirname(__file__), "assets", "korean_dict_jieba.dict")
    40→        ko_scores = {}
    41→        with open(ko_dict_path, "r", encoding="utf-8") as f:
    42→            for line in f:
    43→                line = line.strip()
    44→                if not line:
    45→                    continue
    46→                word = line.split()[0]
    47→                ko_scores[word] = 1.0
    48→        self.ko_score = ko_scores
    49→        self.ko_tokenizer = None
    50→
    51→    def is_kept_char(self, ch: str) -> bool:
    52→        if ch == "'":
    53→            return True
    54→        cat = unicodedata.category(ch)
    55→        if cat.startswith("L") or cat.startswith("N"):
    56→            return True
    57→        return False
    58→
    59→    def clean_token(self, token: str) -> str:
    60→        return "".join(ch for ch in token if self.is_kept_char(ch))
    61→
    62→    def is_cjk_char(self, ch: str) -> bool:
    63→        code = ord(ch)
    64→        return (
    65→            0x4E00 <= code <= 0x9FFF   # CJK Unified Ideographs
    66→            or 0x3400 <= code <= 0x4DBF  # Extension A
    67→            or 0x20000 <= code <= 0x2A6DF  # Extension B
    68→            or 0x2A700 <= code <= 0x2B73F  # Extension C
    69→            or 0x2B740 <= code <= 0x2B81F  # Extension D
    70→            or 0x2B820 <= code <= 0x2CEAF  # Extension E
    71→            or 0xF900 <= code <= 0xFAFF    # Compatibility Ideographs
    72→        )
    73→
    74→    def tokenize_chinese_mixed(self, text: str) -> List[str]:
    75→        tokens: List[str] = []
    76→        current_latin: List[str] = []
    77→
    78→        def flush_latin():
    79→            nonlocal current_latin
    80→            if current_latin:
    81→                token = "".join(current_latin)
    82→                cleaned = self.clean_token(token)
    83→                if cleaned:
    84→                    tokens.append(cleaned)
    85→                current_latin = []
    86→
    87→        for ch in text:
    88→            if self.is_cjk_char(ch):
    89→                flush_latin()
    90→                tokens.append(ch)
    91→            else:
    92→                if self.is_kept_char(ch):
    93→                    current_latin.append(ch)
    94→                else:
    95→                    flush_latin()
    96→
    97→        flush_latin()
    98→
    99→        return tokens
   100→
   101→    def tokenize_japanese(self, text: str) -> List[str]:
   102→        words = nagisa.tagging(text).words
   103→        tokens: List[str] = []
   104→        for w in words:
   105→            cleaned = self.clean_token(w)
   106→            if cleaned:
   107→                tokens.append(cleaned)
   108→        return tokens
   109→
   110→    def tokenize_korean(self, ko_tokenizer, text: str) -> List[str]:
   111→        raw_tokens = ko_tokenizer.tokenize(text)
   112→        tokens: List[str] = []
   113→        for w in raw_tokens:
   114→            w_clean = self.clean_token(w)
   115→            if w_clean:
   116→                tokens.append(w_clean)
   117→        return tokens
   118→
   119→    def split_segment_with_chinese(self, seg: str) -> List[str]:
   120→        tokens: List[str] = []
   121→        buf: List[str] = []
   122→
   123→        def flush_buf():
   124→            nonlocal buf
   125→            if buf:
   126→                tokens.append("".join(buf))
   127→                buf = []
   128→
   129→        for ch in seg:
   130→            if self.is_cjk_char(ch):
   131→                flush_buf()
   132→                tokens.append(ch)
   133→            else:
   134→                buf.append(ch)
   135→
   136→        flush_buf()
   137→        return tokens
   138→
   139→    def tokenize_space_lang(self, text: str) -> List[str]:
   140→        tokens: List[str] = []
   141→        for seg in text.split():
   142→            cleaned = self.clean_token(seg)
   143→            if cleaned:
   144→                tokens.extend(self.split_segment_with_chinese(cleaned))
   145→        return tokens
   146→
   147→    def fix_timestamp(self, data) -> List[int]:
   148→        data = data.tolist()
   149→        n = len(data)
   150→


     1→# coding=utf-8
     2→# Copyright 2026 The Alibaba Qwen team.
     3→# SPDX-License-Identifier: Apache-2.0
     4→#
     5→# Licensed under the Apache License, Version 2.0 (the "License");
     6→# you may not use this file except in compliance with the License.
     7→# You may obtain a copy of the License at
     8→#
     9→#     http://www.apache.org/licenses/LICENSE-2.0
    10→#
    11→# Unless required by applicable law or agreed to in writing, software
    12→# distributed under the License is distributed on an "AS IS" BASIS,
    13→# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14→# See the License for the specific language governing permissions and
    15→# limitations under the License.
    16→"""
    17→qwen_asr: Qwen3-ASR package.
    18→"""
    19→
    20→from .inference.qwen3_asr import Qwen3ASRModel
    21→from .inference.qwen3_forced_aligner import Qwen3ForcedAligner
    22→
    23→from .inference.utils import parse_asr_output
    24→
    25→__all__ = ["__version__"]
    26→


1306-        "attentions": Qwen3ASRThinkerTextAttention,
1307-    }
1308-    config_class = Qwen3ASRConfig
1309-
1310-
1311:class Qwen3ASRForConditionalGeneration(Qwen3ASRPreTrainedModel, GenerationMixin):
1312-    config_class = Qwen3ASRConfig
1313-
1314-    def __init__(self, config: Qwen3ASRConfig):
1315-        super().__init__(config)
1316-        self.config = config


  1311→class Qwen3ASRForConditionalGeneration(Qwen3ASRPreTrainedModel, GenerationMixin):
  1312→    config_class = Qwen3ASRConfig
  1313→
  1314→    def __init__(self, config: Qwen3ASRConfig):
  1315→        super().__init__(config)
  1316→        self.config = config
  1317→
  1318→        self.thinker = Qwen3ASRThinkerForConditionalGeneration._from_config(config.thinker_config)
  1319→        self.post_init()
  1320→
  1321→    def get_support_languages(self):
  1322→        return self.config.support_languages
  1323→
  1324→    @torch.no_grad()
  1325→    def generate(
  1326→        self,
  1327→        input_ids: Optional[torch.Tensor] = None,
  1328→        max_new_tokens: int = 4096,
  1329→        eos_token_id: int | list[int] = [151645, 151643],
  1330→        **kwargs,
  1331→    ):
  1332→        shared_kwargs = {}
  1333→        thinker_kwargs = {
  1334→            "max_new_tokens": max_new_tokens,
  1335→            "eos_token_id": eos_token_id,
  1336→        }
  1337→
  1338→        for key, value in kwargs.items():
  1339→            # Process special input values
  1340→            if key == "feature_attention_mask":
  1341→                thinker_kwargs[key] = value
  1342→            elif key in ("input_features", "attention_mask"):
  1343→                thinker_kwargs[key] = value
  1344→            # Put other key to shared kwargs
  1345→            else:
  1346→                shared_kwargs[key] = value
  1347→
  1348→        # Merge kwargs
  1349→        for key, value in shared_kwargs.items():
  1350→            if key not in thinker_kwargs:
  1351→                thinker_kwargs[key] = value
  1352→
  1353→        thinker_result = self.thinker.generate(input_ids=input_ids, return_dict_in_generate=True, **thinker_kwargs)
  1354→
  1355→        return thinker_result
  1356→
  1357→
  1358→__all__ = [
  1359→    "Qwen3ASRForConditionalGeneration",
  1360→    "Qwen3ASRThinkerTextModel",
  1361→    "Qwen3ASRThinkerForConditionalGeneration",
  1362→    "Qwen3ASRPreTrainedModel",
  1363→    "Qwen3ASRPreTrainedModelForConditionalGeneration",
  1364→    "Qwen3ASRThinkerTextPreTrainedModel",
  1365→]
  1366→


   490→    def _infer_asr_transformers(
   491→        self,
   492→        contexts: List[str],
   493→        wavs: List[np.ndarray],
   494→        languages: List[Optional[str]],
   495→    ) -> List[str]:
   496→        outs: List[str] = []
   497→
   498→        texts = [self._build_text_prompt(context=c, force_language=fl) for c, fl in zip(contexts, languages)]
   499→
   500→        batch_size = self.max_inference_batch_size
   501→        if batch_size is None or batch_size < 0:
   502→            batch_size = len(texts)
   503→
   504→        for i in range(0, len(texts), batch_size):
   505→            sub_text = texts[i : i + batch_size]
   506→            sub_wavs = wavs[i : i + batch_size]
   507→            inputs = self.processor(text=sub_text, audio=sub_wavs, return_tensors="pt", padding=True)
   508→            inputs = inputs.to(self.model.device).to(self.model.dtype)
   509→
   510→            text_ids = self.model.generate(**inputs, max_new_tokens=self.max_new_tokens)
   511→
   512→            decoded = self.processor.batch_decode(
   513→                text_ids.sequences[:, inputs["input_ids"].shape[1]:],
   514→                skip_special_tokens=True,
   515→                clean_up_tokenization_spaces=False,
   516→            )
   517→            outs.extend(list(decoded))
   518→
   519→        return outs
   520→
   521→    def _infer_asr_vllm(
   522→        self,
   523→        contexts: List[str],
   524→        wavs: List[np.ndarray],
   525→        languages: List[Optional[str]],
   526→    ) -> List[str]:
   527→        inputs: List[Dict[str, Any]] = []
   528→        for c, w, fl in zip(contexts, wavs, languages):
   529→            prompt = self._build_text_prompt(context=c, force_language=fl)
   530→            inputs.append({"prompt": prompt, "multi_modal_data": {"audio": [w]}})
   531→
   532→        outs: List[str] = []
   533→        for batch in chunk_list(inputs, self.max_inference_batch_size):
   534→            outputs = self.model.generate(batch, sampling_params=self.sampling_params, use_tqdm=False)
   535→            for o in outputs:
   536→                outs.append(o.outputs[0].text)
   537→        return outs
   538→
   539→    def _offset_align_result(self, result: Any, offset_sec: float) -> Any:
   540→        """
   541→        Apply time offset to a ForcedAlignResult-like object.
   542→
   543→        This function assumes:
   544→          - result has attribute `.items` which is a list of items with start_time/end_time in seconds.
   545→          - dataclasses are frozen in upstream implementation, so we reconstruct by type.
   546→
   547→        Args:
   548→            result: ForcedAlignResult
   549→            offset_sec: Offset in seconds
   550→
   551→        Returns:
   552→            ForcedAlignResult: New object with shifted timestamps.
   553→        """
   554→        if result is None:
   555→            return None
   556→        items = []
   557→        for it in result.items:
   558→            items.append(type(it)(text=it.text,
   559→                                  start_time=round(it.start_time + offset_sec, 3),
   560→                                  end_time=round(it.end_time + offset_sec, 3)))
   561→        return type(result)(items=items)
   562→
   563→    def _merge_align_results(self, results: List[Any]) -> Optional[Any]:
   564→        """
   565→        Merge multiple ForcedAlignResult objects into a single one by concatenating items.
   566→
   567→        Args:
   568→            results: List of ForcedAlignResult
   569→
   570→        Returns:
   571→            ForcedAlignResult or None
   572→        """
   573→        if not results:
   574→            return None
   575→        all_items = []
   576→        for r in results:
   577→            if r is None:
   578→                continue
   579→            all_items.extend(list(r.items))
   580→        if not all_items:
   581→            return None
   582→        return type(results[0])(items=all_items)
   583→
   584→    def init_streaming_state(
   585→        self,
   586→        context: str = "",
   587→        language: Optional[str] = None,
   588→        unfixed_chunk_num: int = 2,
   589→        unfixed_token_num: int = 5,
   590→        chunk_size_sec: float = 2.0,
   591→    ) -> ASRStreamingState:
   592→        """
   593→        Initialize streaming ASR state for a single stream.
   594→
   595→        Notes:
   596→            - Streaming ASR is supported ONLY for vLLM backend.
   597→            - Streaming ASR does NOT support timestamps (forced aligner is not used).
   598→            - Batch inference is NOT supported.
   599→
   600→        Args:
   601→            context:
   602→                Context string.
   603→            language:
   604→                Optional forced language. If provided, it must be in supported languages.
   605→                Same behavior as transcribe(): forces text-only output via prompt suffix.
   606→            unfixed_chunk_num:
   607→                For the first N chunks, do not use previous output as prefix prompt (reset prefix to "").
   608→            unfixed_token_num:
   609→                Roll back the last K tokens from accumulated output when using it as prefix prompt
   610→                after unfixed_chunk_num.
   611→            chunk_size_sec:
   612→                Chunk size in seconds (audio is 16k PCM). The function will internally convert it
   613→                to sample count at 16kHz.
   614→
   615→        Returns:
   616→            ASRStreamingState: Mutable state object to be passed to streaming_transcribe() and
   617→            finish_streaming_transcribe().
   618→
   619→        Raises:
   620→            ValueError:
   621→                - If backend is not "vllm".
   622→                - If chunk_size_sec <= 0.
   623→                - If forced language is invalid (same validation rules as transcribe()).
   624→        """
   625→        if self.backend != "vllm":
   626→            raise ValueError("Streaming ASR is supported only for vLLM backend (backend='vllm').")
   627→        if chunk_size_sec is None or float(chunk_size_sec) <= 0:
   628→            raise ValueError(f"chunk_size_sec must be > 0, got: {chunk_size_sec}")
   629→
   630→        force_language = None
   631→        if language is not None and str(language).strip() != "":
   632→            ln = normalize_language_name(str(language))
   633→            validate_language(ln)
   634→            force_language = ln
   635→
   636→        chunk_size_samples = int(round(float(chunk_size_sec) * SAMPLE_RATE))
   637→        chunk_size_samples = max(1, chunk_size_samples)
   638→
   639→        prompt_raw = self._build_text_prompt(context=context, force_language=force_language)


   200→)
   201→
   202→for r in results:
   203→    print(r.language, r.text, r.time_stamps[0])
   204→```
   205→
   206→For more detailed usage examples, please refer to the [example code](https://github.com/QwenLM/Qwen3-ASR/blob/main/examples/example_qwen3_asr_transformers.py) for the transformers backend.
   207→
   208→#### vLLM Backend
   209→
   210→If you want the fastest inference speed with Qwen3-ASR, we strongly recommend using the vLLM backend by initializing the model with `Qwen3ASRModel.LLM(...)`. Example code is provided below. Note that you must install it via `pip install -U qwen-asr[vllm]`. If you want the model to output timestamps, it’s best to install FlashAttention via `pip install -U flash-attn --no-build-isolation` to speed up inference for the forced aligner model. Remember to wrap your code under `if __name__ == '__main__':` to avoid the `spawn` error described in [vLLM Troubleshooting](https://docs.vllm.ai/en/latest/usage/troubleshooting/#python-multiprocessing).
   211→
   212→```python
   213→import torch
   214→from qwen_asr import Qwen3ASRModel
   215→
   216→if __name__ == '__main__':
   217→    model = Qwen3ASRModel.LLM(
   218→        model="Qwen/Qwen3-ASR-1.7B",
   219→        gpu_memory_utilization=0.7,
   220→        max_inference_batch_size=128, # Batch size limit for inference. -1 means unlimited. Smaller values can help avoid OOM.
   221→        max_new_tokens=4096, # Maximum number of tokens to generate. Set a larger value for long audio input.
   222→        forced_aligner="Qwen/Qwen3-ForcedAligner-0.6B",
   223→        forced_aligner_kwargs=dict(
   224→            dtype=torch.bfloat16,
   225→            device_map="cuda:0",
   226→            # attn_implementation="flash_attention_2",
   227→        ),
   228→    )
   229→
   230→    results = model.transcribe(
   231→        audio=[
   232→        "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_zh.wav",
   233→        "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav",
   234→        ],
   235→        language=["Chinese", "English"], # can also be set to None for automatic language detection
   236→        return_time_stamps=True,
   237→    )
   238→
   239→    for r in results:
   240→        print(r.language, r.text, r.time_stamps[0])
   241→```
   242→
   243→For more detailed usage examples, please refer to the [example code](https://github.com/QwenLM/Qwen3-ASR/blob/main/examples/example_qwen3_asr_vllm.py) for the vLLM backend. In addition, you can start a vLLM server via the `qwen-asr-serve` command, which is a wrapper around `vllm serve`. You can pass any arguments supported by `vllm serve`, for example:
   244→
   245→```bash
   246→qwen-asr-serve Qwen/Qwen3-ASR-1.7B --gpu-memory-utilization 0.8 --host 0.0.0.0 --port 8000
   247→```
   248→
   249→And send requests to the server via:
   250→
   251→```python
   252→import requests
   253→
   254→url = "http://localhost:8000/v1/chat/completions"
   255→headers = {"Content-Type": "application/json"}
   256→
   257→data = {
   258→    "messages": [
   259→        {
   260→            "role": "user",
   261→            "content": [
   262→                {
   263→                    "type": "audio_url",
   264→                    "audio_url": {
   265→                        "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav"
   266→                    },
   267→                }
   268→            ],
   269→        }
   270→    ]
   271→}
   272→
   273→response = requests.post(url, headers=headers, json=data, timeout=300)
   274→response.raise_for_status()
   275→content = response.json()['choices'][0]['message']['content']
   276→print(content)
   277→
   278→# parse ASR output if you want
   279→from qwen_asr import parse_asr_output
   280→language, text = parse_asr_output(content)
   281→print(language)
   282→print(text)
   283→```
   284→
   285→#### Streaming Inference
   286→
   287→Qwen3-ASR fully supports streaming inference. Currently, streaming inference is only available with the vLLM backend. Note that streaming inference does not support batch inference or returning timestamps. Please refer to the [example code](https://github.com/QwenLM/Qwen3-ASR/blob/main/examples/example_qwen3_asr_vllm_streaming.py) for details. You can also launch a streaming web demo through the [guide](#streaming-demo) to experience Qwen3-ASR’s streaming transcription capabilities.
   288→
   289→#### ForcedAligner Usage
   290→
   291→`Qwen3-ForcedAligner-0.6B` can align text–speech pairs and return word or character level timestamps. Here is an example of using the forced aligner directly:
   292→
   293→```python
   294→import torch
   295→from qwen_asr import Qwen3ForcedAligner
   296→
   297→model = Qwen3ForcedAligner.from_pretrained(
   298→    "Qwen/Qwen3-ForcedAligner-0.6B",
   299→    dtype=torch.bfloat16,
   300→    device_map="cuda:0",
   301→    # attn_implementation="flash_attention_2",
   302→)
   303→
   304→results = model.align(
   305→    audio="https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_zh.wav",
   306→    text="甚至出现交易几乎停滞的情况。",
   307→    language="Chinese",
   308→)
   309→
   310→print(results[0])
   311→print(results[0][0].text, results[0][0].start_time, results[0][0].end_time)
   312→```
   313→
   314→In addition, the forced aligner supports local paths / URLs / base64 data / `(np.ndarray, sr)` inputs and batch inference. Please refer to the [example code](https://github.com/QwenLM/Qwen3-ASR/blob/main/examples/example_qwen3_forced_aligner.py) for details.
   315→
   316→### DashScope API Usage
   317→
   318→To further explore Qwen3-ASR, we encourage you to try our DashScope API for a faster and more efficient experience. For detailed API information and documentation, please refer to the following:
   319→
   320→| API Description | API Documentation (Mainland China) | API Documentation (International) |
   321→|------------------|-----------------------------------|------------------------------------|
   322→| Real-time API for Qwen3-ASR. | [https://help.aliyun.com/zh/model-studio/qwen-real-time-speech-recognition](https://help.aliyun.com/zh/model-studio/qwen-real-time-speech-recognition) | [https://www.alibabacloud.com/help/en/model-studio/qwen-real-time-speech-recognition](https://www.alibabacloud.com/help/en/model-studio/qwen-real-time-speech-recognition) |
   323→| FileTrans API for Qwen3-ASR. | [https://help.aliyun.com/zh/model-studio/qwen-speech-recognition](https://help.aliyun.com/zh/model-studio/qwen-speech-recognition) | [https://www.alibabacloud.com/help/en/model-studio/qwen-speech-recognition](https://www.alibabacloud.com/help/en/model-studio/qwen-speech-recognition) |
   324→
   325→
   326→## Launch Local Web UI Demo
   327→
   328→### Gradio Demo
   329→
   330→To launch the Qwen3-ASR web UI gradio demo, install the `qwen-asr` package and run `qwen-asr-demo`. Use the command below for help:
   331→
   332→```bash
   333→qwen-asr-demo --help
   334→```
   335→
   336→To launch the demo, you can use the following commands:
   337→
   338→```bash
   339→# Transformers backend
   340→qwen-asr-demo \
   341→  --asr-checkpoint Qwen/Qwen3-ASR-1.7B \
   342→  --backend transformers \
   343→  --cuda-visible-devices 0 \
   344→  --ip 0.0.0.0 --port 8000
   345→
   346→# Transformers backend + Forced Aligner (enable timestamps)
   347→qwen-asr-demo \
   348→  --asr-checkpoint Qwen/Qwen3-ASR-1.7B \
   349→  --aligner-checkpoint Qwen/Qwen3-ForcedAligner-0.6B \
   350→  --backend transformers \
   351→  --cuda-visible-devices 0 \
   352→  --backend-kwargs '{"device_map":"cuda:0","dtype":"bfloat16","max_inference_batch_size":8,"max_new_tokens":256}' \
   353→  --aligner-kwargs '{"device_map":"cuda:0","dtype":"bfloat16"}' \
   354→  --ip 0.0.0.0 --port 8000
   355→
   356→# vLLM backend + Forced Aligner (enable timestamps)
   357→qwen-asr-demo \
   358→  --asr-checkpoint Qwen/Qwen3-ASR-1.7B \
   359→  --aligner-checkpoint Qwen/Qwen3-ForcedAligner-0.6B \
   360→  --backend vllm \
   361→  --cuda-visible-devices 0 \
   362→  --backend-kwargs '{"gpu_memory_utilization":0.7,"max_inference_batch_size":8,"max_new_tokens":2048}' \
   363→  --aligner-kwargs '{"device_map":"cuda:0","dtype":"bfloat16"}' \
   364→  --ip 0.0.0.0 --port 8000
   365→```
   366→
   367→Then open `http://<your-ip>:8000`, or access it via port forwarding in tools like VS Code.
   368→
   369→#### Backend Notes
   370→
   371→This demo supports two backends: transformers and vLLM. All backend-specific initialization parameters should be passed via `--backend-kwargs` as a JSON dict. If not provided, the demo will use sensible defaults.
   372→
   373→```bash
   374→# Example: override transformers init args with flash attention
   375→--backend-kwargs '{"device_map":"cuda:0","dtype":"bfloat16","attn_implementation":"flash_attention_2"}'
   376→
   377→# Example: override vLLM init args with 65% GPU memory
   378→--backend-kwargs '{"gpu_memory_utilization":0.65}'
   379→```
   380→
   381→#### CUDA Device Notes
   382→
   383→Because vLLM does not follow `cuda:0` style device selection, this demo selects GPUs by setting `CUDA_VISIBLE_DEVICES` via `--cuda-visible-devices`.
   384→
   385→```bash
   386→# Use GPU 0
   387→--cuda-visible-devices 0
   388→
   389→# Use GPU 1
   390→--cuda-visible-devices 1
   391→```
   392→
   393→#### Timestamps Notes
   394→
   395→Timestamps are only available when `--aligner-checkpoint` is provided. If you launch the demo without a forced aligner, the timestamps UI will be hidden automatically.
   396→
   397→```bash
   398→# No forced aligner
   399→qwen-asr-demo --asr-checkpoint Qwen/Qwen3-ASR-1.7B
   400→
   401→# With forced aligner
   402→qwen-asr-demo \
   403→  --asr-checkpoint Qwen/Qwen3-ASR-1.7B \
   404→  --aligner-checkpoint Qwen/Qwen3-ForcedAligner-0.6B
   405→```
   406→
   407→#### HTTPS Notes
   408→
   409→To avoid browser microphone permission issues after deploying the server, it is recommended/required to run the gradio service over HTTPS (especially when accessed remotely or behind modern browsers/gateways). Use `--ssl-certfile` and `--ssl-keyfile` to enable HTTPS. First, generate a private key and a self-signed certificate (valid for 365 days):
   410→
   411→```bash
   412→openssl req -x509 -newkey rsa:2048 \
   413→  -keyout key.pem -out cert.pem \
   414→  -days 365 -nodes \
   415→  -subj "/CN=localhost"
   416→```
   417→
   418→Then run the demo with HTTPS:
   419→
   420→```bash
   421→qwen-asr-demo \
   422→  --asr-checkpoint Qwen/Qwen3-ASR-1.7B \
   423→  --backend transformers \
   424→  --cuda-visible-devices 0 \
   425→  --ip 0.0.0.0 --port 8000 \
   426→  --ssl-certfile cert.pem \
   427→  --ssl-keyfile key.pem \
   428→  --no-ssl-verify
   429→```
   430→
   431→Then open `https://<your-ip>:8000` to use it. If your browser shows a warning, that’s expected for self-signed certificates. For production, use a real certificate.
   432→
   433→### Streaming Demo
   434→
   435→To experience Qwen3-ASR’s streaming transcription capability in a web UI, we provide a minimal Flask-based streaming demo. The demo captures microphone audio in the browser, resamples it to 16,000 Hz, and continuously pushes PCM chunks to the model. Run the demo with the following command:
   436→
   437→```bash
   438→qwen-asr-demo-streaming \
   439→  --asr-model-path Qwen/Qwen3-ASR-1.7B \
   440→  --gpu-memory-utilization 0.9 \
   441→  --host 0.0.0.0 \
   442→  --port 8000
   443→```
   444→
   445→Then open `http://<your-ip>:8000`, or access it via port forwarding in tools like VS Code.
   446→
   447→## Deployment with vLLM
   448→
   449→vLLM officially provides day-0 model support for Qwen3-ASR for efficient inference.
   450→
   451→### Installation
   452→You can run Qwen3-ASR with vLLM nightly wheel or docker image. To install the nightly version of vLLM, we recommend using `uv` as the environment manager
   453→```bash
   454→uv venv
   455→source .venv/bin/activate
   456→uv pip install -U vllm --pre \
   457→    --extra-index-url https://wheels.vllm.ai/nightly/cu129 \
   458→    --extra-index-url https://download.pytorch.org/whl/cu129 \
   459→    --index-strategy unsafe-best-match
   460→uv pip install "vllm[audio]" # For additional audio dependencies
   461→```
   462→
   463→### Online Serving
   464→You can easily deploy Qwen3-ASR with vLLM by running the following command
   465→```bash
   466→vllm serve Qwen/Qwen3-ASR-1.7B
   467→```
   468→After the model server is successfully deployed, you can interact with it in multiple ways.
   469→
   470→#### Using OpenAI SDK
   471→```python
   472→import base64
   473→import httpx
   474→from openai import OpenAI
   475→
   476→# Initialize client
   477→client = OpenAI(
   478→    base_url="http://localhost:8000/v1",
   479→    api_key="EMPTY"
   480→)
   481→
   482→# Create multimodal chat completion request
   483→response = client.chat.completions.create(
   484→    model="Qwen/Qwen3-ASR-1.7B",
   485→    messages=[
   486→        {
   487→            "role": "user",
   488→            "content": [
   489→                {
   490→                    "type": "audio_url",
   491→                    "audio_url": {
   492→                        {"url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav"}
   493→                    }
   494→                }
   495→            ]
   496→        }
   497→    ],
   498→)
   499→


find /home/ubuntu/training/qwen3-asr-1.7b-phase2-sft -type f -name "*.py" | xargs grep -l "eval\|benchmark\|test" | head -10


find /home/ubuntu/training/qwen3-asr-1.7b-phase2-sft -type f \( -name "*eval*" -o -name "*bench*" \) | head -10


449-
450-
451:class Qwen3ASRAudioAttention(nn.Module):
452-    """Multi-headed attention from 'Attention Is All You Need' paper"""
453-
--
521-
522-
523:class Qwen3ASRAudioEncoderLayer(GradientCheckpointingLayer):
524-    def __init__(self, config: Qwen3ASRAudioEncoderConfig):
525-        super().__init__()
--
601-    """
602-)
603:class Qwen3ASRAudioEncoder(Qwen3ASRPreTrainedModel):
604-    config: Qwen3ASRAudioEncoderConfig
605-    main_input_name = "input_features"


   603→class Qwen3ASRAudioEncoder(Qwen3ASRPreTrainedModel):
   604→    config: Qwen3ASRAudioEncoderConfig
   605→    main_input_name = "input_features"
   606→    _no_split_modules = ["Qwen3ASRAudioEncoderLayer"]
   607→    _supports_sdpa = True
   608→
   609→    def __init__(self, config: Qwen3ASRAudioEncoderConfig):
   610→        super().__init__(config)
   611→        self.dropout = config.dropout
   612→
   613→        embed_dim = config.d_model
   614→        self.num_mel_bins = config.num_mel_bins
   615→        self.max_source_positions = config.max_source_positions
   616→        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
   617→        self.n_window = config.n_window
   618→        self.positional_embedding = SinusoidsPositionEmbedding(self.max_source_positions, embed_dim)
   619→        self.layers = nn.ModuleList([Qwen3ASRAudioEncoderLayer(config) for _ in range(config.encoder_layers)])
   620→        self.ln_post = nn.LayerNorm(config.d_model)
   621→        self.gradient_checkpointing = False
   622→        self.conv2d1 = nn.Conv2d(1, config.downsample_hidden_size, 3, 2, padding=1)
   623→        self.conv2d2 = nn.Conv2d(config.downsample_hidden_size, config.downsample_hidden_size, 3, 2, padding=1)
   624→        self.conv2d3 = nn.Conv2d(config.downsample_hidden_size, config.downsample_hidden_size, 3, 2, padding=1)
   625→        self.conv_out = nn.Linear(
   626→            config.downsample_hidden_size * ((((config.num_mel_bins + 1) // 2 + 1) // 2 + 1) // 2),
   627→            config.d_model,
   628→            bias=False,
   629→        )
   630→        self.proj1 = nn.Linear(config.d_model, config.d_model)
   631→        self.act = ACT2FN[config.activation_function]
   632→        self.proj2 = nn.Linear(config.d_model, config.output_dim)
   633→        self.n_window_infer = self.config.n_window_infer
   634→        self.conv_chunksize = self.config.conv_chunksize
   635→        # Initialize weights and apply final processing
   636→        self.post_init()
   637→
   638→    def _freeze_parameters(self):
   639→        for param in self.parameters():
   640→            param.requires_grad = False
   641→        self._requires_grad = False
   642→
   643→    def get_input_embeddings(self) -> nn.Module:
   644→        return self.conv1
   645→
   646→    def set_input_embeddings(self, value: nn.Module):
   647→        self.conv1 = value
   648→
   649→    def _prepare_attention_mask(self, inputs_tensor: torch.Tensor, cu_seqlens: torch.Tensor) -> torch.Tensor:
   650→        # Flash Attention 2 doesn't need a 4D mask and relies on `cu_seqlens/max_seqlen`
   651→        # NOTE: the created attention masl only approximates the ragged FA2 attention by
   652→        # allowing bidirectional attention within `cu_seqlens` blocks, and not attending between
   653→        # blocks. Though it will not be a 100% match for FA2's `varlen` path
   654→        if self.config._attn_implementation == "flash_attention_2":
   655→            return None
   656→
   657→        seq_length = inputs_tensor.shape[0]
   658→        attention_mask = torch.full(
   659→            [1, 1, seq_length, seq_length],
   660→            torch.finfo(inputs_tensor.dtype).min,
   661→            device=inputs_tensor.device,
   662→            dtype=inputs_tensor.dtype,
   663→        )
   664→        for i in range(1, len(cu_seqlens)):
   665→            attention_mask[..., cu_seqlens[i - 1] : cu_seqlens[i], cu_seqlens[i - 1] : cu_seqlens[i]] = 0
   666→        return attention_mask
   667→
   668→    @auto_docstring
   669→    def forward(
   670→        self,
   671→        input_features,
   672→        feature_lens=None,
   673→        aftercnn_lens=None,
   674→    ):
   675→        r"""
   676→        feature_lens (`torch.LongTensor` of shape `(batch_size,)`):
   677→            mel length
   678→        aftercnn_lens (`torch.LongTensor` of shape `(batch_size,)`):
   679→            mel length after cnn
   680→        """
   681→        aftercnn_lens = _get_feat_extract_output_lengths(feature_lens)
   682→        chunk_num = torch.ceil(feature_lens / (self.n_window * 2)).long()
   683→
   684→        chunk_lengths = torch.tensor(
   685→            [self.n_window * 2] * chunk_num.sum(),
   686→            dtype=torch.long,
   687→            device=feature_lens.device,
   688→        )
   689→        tail_chunk_index = F.pad(chunk_num, (1, 0), value=-1).cumsum(0)[1:]
   690→        chunk_lengths[tail_chunk_index] = feature_lens % (self.n_window * 2)
   691→        chunk_lengths[chunk_lengths == 0] = self.n_window * 2
   692→
   693→        chunk_list = input_features.T.split(chunk_lengths.tolist(), dim=0)
   694→        padded_feature = nn.utils.rnn.pad_sequence(chunk_list, batch_first=True).transpose(1, 2)
   695→        feature_lens_after_cnn = _get_feat_extract_output_lengths(chunk_lengths)
   696→        padded_mask_after_cnn = nn.utils.rnn.pad_sequence(
   697→            [torch.ones(length, dtype=torch.bool, device=padded_feature.device) for length in feature_lens_after_cnn],
   698→            batch_first=True,
   699→        )
   700→        padded_feature = padded_feature.unsqueeze(1)
   701→        # Split to chunk to avoid OOM during convolution
   702→        padded_embeds = []
   703→        for chunk in padded_feature.split(self.conv_chunksize, dim=0):
   704→            padded_embed = F.gelu(self.conv2d1(chunk))
   705→            padded_embed = F.gelu(self.conv2d2(padded_embed))
   706→            padded_embed = F.gelu(self.conv2d3(padded_embed))
   707→            padded_embeds.append(padded_embed)
   708→        padded_embed = torch.cat(padded_embeds, dim=0)
   709→        b, c, f, t = padded_embed.size()
   710→        padded_embed = self.conv_out(padded_embed.permute(0, 3, 1, 2).contiguous().view(b, t, c * f))
   711→
   712→        positional_embedding = (
   713→            self.positional_embedding.positional_embedding[: padded_embed.shape[1], :]
   714→            .unsqueeze(0)
   715→            .to(padded_embed.dtype)
   716→        )
   717→        padded_embed = padded_embed + positional_embedding
   718→        hidden_states = padded_embed[padded_mask_after_cnn]
   719→        cu_chunk_lens = [0]
   720→        window_aftercnn = padded_mask_after_cnn.shape[-1] * (self.n_window_infer // (self.n_window * 2))
   721→        for cnn_len in aftercnn_lens:
   722→            cu_chunk_lens += [window_aftercnn] * (cnn_len // window_aftercnn)
   723→            remainder = cnn_len % window_aftercnn
   724→            if remainder != 0:
   725→                cu_chunk_lens += [remainder]
   726→        cu_seqlens = torch.tensor(cu_chunk_lens, device=aftercnn_lens.device).cumsum(-1, dtype=torch.int32)
   727→
   728→        for encoder_layer in self.layers:
   729→            layer_outputs = encoder_layer(
   730→                hidden_states,
   731→                cu_seqlens,
   732→            )
   733→
   734→            hidden_states = layer_outputs[0]
   735→
   736→        hidden_states = self.ln_post(hidden_states)
   737→        hidden_states = self.proj1(hidden_states)
   738→        hidden_states = self.act(hidden_states)
   739→        hidden_states = self.proj2(hidden_states)
   740→        return BaseModelOutput(last_hidden_state=hidden_states)
   741→
   742→    def padded_and_mask_function(self, tensor_list, tensor_len, padding_value=0, padding_side="right"):
   743→        """
   744→        Pads a sequence of tensors to their maximum length on indicated `padding_side`.
   745→        Then prepares a mask so that pad tokens are not attended to.
   746→        """
   747→        max_len = tensor_len.max()
   748→        dim = tensor_list[0].shape[0]
   749→        padded_tensor = torch.full(
   750→            size=(len(tensor_list), dim, max_len),
   751→            fill_value=padding_value,
   752→            dtype=self.dtype,


     1→# SPDX-License-Identifier: Apache-2.0
     2→# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
     3→# Copyright 2026 The Qwen team.
     4→# Copyright 2023 The vLLM team.
     5→# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
     6→#
     7→# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
     8→# and OPT implementations in this library. It has been modified from its
     9→# original forms to accommodate minor architectural differences compared
    10→# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
    11→#
    12→# Licensed under the Apache License, Version 2.0 (the "License");
    13→# you may not use this file except in compliance with the License.
    14→# You may obtain a copy of the License at
    15→#
    16→#     http://www.apache.org/licenses/LICENSE-2.0
    17→#
    18→# Unless required by applicable law or agreed to in writing, software
    19→# distributed under the License is distributed on an "AS IS" BASIS,
    20→# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    21→# See the License for the specific language governing permissions and
    22→# limitations under the License.
    23→"""Inference-only Qwen3-ASR model."""
    24→
    25→from collections.abc import Iterable, Mapping, Sequence
    26→from typing import Any, Literal, cast
    27→
    28→import numpy as np
    29→import torch
    30→import torch.nn as nn
    31→import torch.nn.functional as F
    32→from transformers.feature_extraction_utils import BatchFeature
    33→from transformers.models.whisper import WhisperFeatureExtractor
    34→
    35→from vllm.config import MultiModalConfig, ModelConfig, SpeechToTextConfig, VllmConfig
    36→from vllm.config.multimodal import BaseDummyOptions
    37→from vllm.distributed import get_tensor_model_parallel_world_size
    38→from vllm.inputs.data import PromptType
    39→from vllm.logger import init_logger
    40→from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
    41→from vllm.model_executor.layers.attention.mm_encoder_attention import (
    42→    MMEncoderAttention,
    43→)
    44→from vllm.model_executor.layers.linear import (
    45→    ColumnParallelLinear,
    46→    QKVParallelLinear,
    47→    RowParallelLinear,
    48→)
    49→from vllm.model_executor.model_loader.weight_utils import default_weight_loader
    50→from vllm.model_executor.models.interfaces import (
    51→    MultiModalEmbeddings,
    52→    SupportsMRoPE,
    53→    SupportsMultiModal,
    54→    SupportsPP,
    55→    SupportsTranscription,
    56→)
    57→from vllm.model_executor.models.module_mapping import MultiModelKeys
    58→from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
    59→from vllm.model_executor.models.qwen3_omni_moe_thinker import (
    60→    Qwen2_5OmniAudioFeatureInputs,
    61→    Qwen3OmniMoeThinkerMultiModalProcessor,
    62→)
    63→from vllm.model_executor.models.utils import (
    64→    AutoWeightsLoader,
    65→    WeightsMapper,
    66→    _merge_multimodal_embeddings,
    67→    maybe_prefix,
    68→)
    69→from vllm.model_executor.models.whisper import ISO639_1_SUPPORTED_LANGS
    70→from vllm.multimodal import MULTIMODAL_REGISTRY
    71→from vllm.multimodal.inputs import (
    72→    AudioItem,
    73→    ModalityData,
    74→    MultiModalDataDict,
    75→    MultiModalFeatureSpec,
    76→    MultiModalFieldConfig,
    77→    MultiModalKwargsItems,
    78→)
    79→from vllm.multimodal.parse import (
    80→    AudioProcessorItems,
    81→    DictEmbeddingItems,
    82→    ModalityDataItems,
    83→    MultiModalDataItems,
    84→    MultiModalDataParser,
    85→)
    86→from vllm.multimodal.processing import (
    87→    BaseProcessingInfo,
    88→    PromptReplacement,
    89→    PromptUpdate,
    90→)
    91→from vllm.sequence import IntermediateTensors
    92→from vllm.v1.attention.backends.registry import AttentionBackendEnum
    93→from vllm.tokenizers import cached_tokenizer_from_config
    94→from vllm.transformers_utils.processor import cached_processor_from_config
    95→from vllm.model_executor.models.vision import (
    96→    get_vit_attn_backend,
    97→)
    98→from ..transformers_backend.configuration_qwen3_asr import (
    99→    Qwen3ASRConfig,
   100→    Qwen3ASRThinkerConfig,
   101→    Qwen3ASRAudioEncoderConfig
   102→)
   103→from ..transformers_backend.processing_qwen3_asr import (
   104→    Qwen3ASRProcessor,
   105→)
   106→
   107→try:
   108→    from vllm.multimodal.profiling import BaseDummyInputsBuilder
   109→except:
   110→    from vllm.multimodal.processing import BaseDummyInputsBuilder
   111→
   112→logger = init_logger(__name__)
   113→
   114→
   115→def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
   116→    input_lengths_leave = input_lengths % 100
   117→    feat_lengths = (input_lengths_leave - 1) // 2 + 1
   118→    output_lengths = (
   119→        ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // 100) * 13
   120→    )
   121→    return output_lengths
   122→
   123→
   124→# ============= Audio Encoder Components =============
   125→
   126→
   127→class SinusoidsPositionEmbedding(nn.Module):
   128→    """Sinusoidal position embedding for audio encoder."""
   129→
   130→    def __init__(self, length: int, channels: int, max_timescale: int = 10000):
   131→        super().__init__()
   132→        self.length = length
   133→        self.channels = channels
   134→        self.max_timescale = max_timescale
   135→
   136→        if channels % 2 != 0:
   137→            raise ValueError("SinusoidsPositionEmbedding needs even channels input")
   138→
   139→        log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
   140→        inv_timescales = torch.exp(
   141→            -log_timescale_increment * torch.arange(channels // 2).float()
   142→        )
   143→        scaled_time = (
   144→            torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
   145→        )
   146→        positional_embedding = torch.cat(
   147→            [torch.sin(scaled_time), torch.cos(scaled_time)], dim=1
   148→        )
   149→        self.register_buffer(
   150→            "positional_embedding", positional_embedding, persistent=False


65→### Model Architecture
    66→
    67→

    68→
    69→

    70→
    71→
    72→### Released Models Description and Download
    73→
    74→Below is an introduction and download information for the Qwen3-ASR models. Please select and download the model that fits your needs.
    75→
    76→| Model | Supported Languages | Supported Dialects | Inference Mode | Audio Types |
    77→|---|---|---|---|---|
    78→| Qwen3-ASR-1.7B & Qwen3-ASR-0.6B | Chinese (zh), English (en), Cantonese (yue), Arabic (ar), German (de), French (fr), Spanish (es), Portuguese (pt), Indonesian (id), Italian (it), Korean (ko), Russian (ru), Thai (th), Vietnamese (vi), Japanese (ja), Turkish (tr), Hindi (hi), Malay (ms), Dutch (nl), Swedish (sv), Danish (da), Finnish (fi), Polish (pl), Czech (cs), Filipino (fil), Persian (fa), Greek (el), Hungarian (hu), Macedonian (mk), Romanian (ro) | Anhui, Dongbei, Fujian, Gansu, Guizhou, Hebei, Henan, Hubei, Hunan, Jiangxi, Ningxia, Shandong, Shaanxi, Shanxi, Sichuan, Tianjin, Yunnan, Zhejiang, Cantonese (Hong Kong accent), Cantonese (Guangdong accent), Wu language, Minnan language. | Offline / Streaming | Speech, Singing Voice, Songs with BGM |
    79→| Qwen3-ForcedAligner-0.6B | Chinese, English, Cantonese, French, German, Italian, Japanese, Korean, Portuguese, Russian, Spanish | -- | NAR | Speech |
    80→
    81→During model loading in the `qwen-asr` package or vLLM, model weights will be downloaded automatically based on the model name. However, if your runtime environment does not allow downloading weights during execution, you can use the following commands to manually download the model weights to a local directory:
    82→
    83→```bash
    84→# Download through ModelScope (recommended for users in Mainland China)
    85→pip install -U modelscope
    86→modelscope download --model Qwen/Qwen3-ASR-1.7B  --local_dir ./Qwen3-ASR-1.7B
    87→modelscope download --model Qwen/Qwen3-ASR-0.6B --local_dir ./Qwen3-ASR-0.6B
    88→modelscope download --model Qwen/Qwen3-ForcedAligner-0.6B --local_dir ./Qwen3-ForcedAligner-0.6B
    89→# Download through Hugging Face
    90→pip install -U "huggingface_hub[cli]"
    91→huggingface-cli download Qwen/Qwen3-ASR-1.7B --local-dir ./Qwen3-ASR-1.7B
    92→huggingface-cli download Qwen/Qwen3-ASR-0.6B --local-dir ./Qwen3-ASR-0.6B
    93→huggingface-cli download Qwen/Qwen3-ForcedAligner-0.6B --local-dir ./Qwen3-ForcedAligner-0.6B
    94→```
    95→
    96→
    97→## Quickstart
    98→
    99→### Environment Setup
   100→
   101→The easiest way to use Qwen3-ASR is to install the `qwen-asr` Python package from PyPI. This will pull in the required runtime dependencies and allow you to load any released Qwen3-ASR model. If you’d like to simplify environment setup further, you can also use our official [Docker image](#docker). The `qwen-asr` package provides two backends: the transformers backend and the vLLM backend. For usage instructions for different backends, please refer to [Python Package Usage](#python-package-usage). We recommend using a **fresh, isolated environment** to avoid dependency conflicts with existing packages. You can create a clean Python 3.12 environment like this:
   102→
   103→```bash
   104→conda create -n qwen3-asr python=3.12 -y
   105→conda activate qwen3-asr
   106→```
   107→
   108→Run the following command to get the minimal installation with transformers-backend support:
   109→
   110→```bash
   111→pip install -U qwen-asr
   112→```
   113→
   114→To enable the vLLM backend for faster inference and streaming support, run:
   115→
   116→```bash
   117→pip install -U qwen-asr[vllm]
   118→```
   119→
   120→If you want to develop or modify the code locally, install from source in editable mode:
   121→
   122→```bash
   123→git clone https://github.com/QwenLM/Qwen3-ASR.git
   124→cd Qwen3-ASR
   125→pip install -e .
   126→# support vLLM backend
   127→# pip install -e ".[vllm]"
   128→```
   129→
   130→Additionally, we recommend using FlashAttention 2 to reduce GPU memory usage and accelerate inference speed, especially for long inputs and large batch sizes.
   131→
   132→```bash
   133→pip install -U flash-attn --no-build-isolation
   134→```
   135→
   136→If your machine has less than 96GB of RAM and lots of CPU cores, run:
   137→
   138→```bash
   139→MAX_JOBS=4 pip install -U flash-attn --no-build-isolation
   140→```
   141→
   142→Also, you should have hardware that is compatible with FlashAttention 2. Read more about it in the official documentation of the [FlashAttention repository](https://github.com/Dao-AILab/flash-attention). FlashAttention 2 can only be used when a model is loaded in `torch.float16` or `torch.bfloat16`.
   143→
   144→### Python Package Usage
   145→
   146→#### Quick Inference
   147→
   148→The `qwen-asr` package provides two backends: **transformers backend** and **vLLM backend**. You can pass audio inputs as a local path, a URL, base64 data, or a `(np.ndarray, sr)` tuple, and run batch inference. To quickly try Qwen3-ASR, you can use `Qwen3ASRModel.from_pretrained(...)` for the transformers backend with the following code:
   149→
   150→```python
   151→import torch
   152→from qwen_asr import Qwen3ASRModel
   153→
   154→model = Qwen3ASRModel.from_pretrained(
   155→    "Qwen/Qwen3-ASR-1.7B",
   156→    dtype=torch.bfloat16,
   157→    device_map="cuda:0",
   158→    # attn_implementation="flash_attention_2",
   159→    max_inference_batch_size=32, # Batch size limit for inference. -1 means unlimited. Smaller values can help avoid OOM.
   160→    max_new_tokens=256, # Maximum number of tokens to generate. Set a larger value for long audio input.
   161→)
   162→
   163→results = model.transcribe(
   164→    audio="https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav",
   165→    language=None, # set "English" to force the language
   166→)
   167→
   168→print(results[0].language)
   169→print(results[0].text)
   170→```
   171→
   172→If you want to return timestamps, pass `forced_aligner` and its init kwargs. Here is an example of batch inference with timestamps output:
   173→
   174→```python
   175→import torch
   176→from qwen_asr import Qwen3ASRModel
   177→
   178→model = Qwen3ASRModel.from_pretrained(
   179→    "Qwen/Qwen3-ASR-1.7B",
   180→    dtype=torch.bfloat16,
   181→    device_map="cuda:0",
   182→    # attn_implementation="flash_attention_2",
   183→    max_inference_batch_size=32, # Batch size limit for inference. -1 means unlimited. Smaller values can help avoid OOM.
   184→    max_new_tokens=256, # Maximum number of tokens to generate. Set a larger value for long audio input.
   185→    forced_aligner="Qwen/Qwen3-ForcedAligner-0.6B",
   186→    forced_aligner_kwargs=dict(
   187→        dtype=torch.bfloat16,
   188→        device_map="cuda:0",
   189→        # attn_implementation="flash_attention_2",
   190→    ),
   191→)
   192→
   193→results = model.transcribe(
   194→    audio=[
   195→      "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_zh.wav",
   196→      "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav",
   197→    ],
   198→    language=["Chinese", "English"], # can also be set to None for automatic language detection
   199→    return_time_stamps=True,
   200→)
   201→
   202→for r in results:
   203→    print(r.language, r.text, r.time_stamps[0])
   204→```
   205→
   206→For more detailed usage examples, please refer to the [example code](https://github.com/QwenLM/Qwen3-ASR/blob/main/examples/example_qwen3_asr_transformers.py) for the transformers backend.
   207→
   208→#### vLLM Backend
   209→
   210→If you want the fastest inference speed with Qwen3-ASR, we strongly recommend using the vLLM backend by initializing the model with `Qwen3ASRModel.LLM(...)`. Example code is provided below. Note that you must install it via `pip install -U qwen-asr[vllm]`. If you want the model to output timestamps, it’s best to install FlashAttention via `pip install -U flash-attn --no-build-isolation` to speed up inference for the forced aligner model. Remember to wrap your code under `if __name__ == '__main__':` to avoid the `spawn` error described in [vLLM Troubleshooting](https://docs.vllm.ai/en/latest/usage/troubleshooting/#python-multiprocessing).
   211→
   212→```python
   213→import torch
   214→from qwen_asr import Qwen3ASRModel