   800→        self.mrope_section = config.rope_scaling.get("mrope_section", [24, 20, 20])
   801→
   802→    def apply_interleaved_mrope(self, freqs, mrope_section):
   803→        """Apply interleaved MRoPE to 3D rotary embeddings.
   804→        Reorganizes frequency layout from chunked [TTT...HHH...WWW] to
   805→        interleaved [THTHWHTHW...TT], preserving frequency continuity.
   806→        args:
   807→            x: (3, bs, seq_len, head_dim // 2)
   808→            mrope_section: (3,)
   809→        returns:
   810→            x_t: (bs, seq_len, head_dim // 2)
   811→        """
   812→        freqs_t = freqs[0]  # just overwrite the first dimension T
   813→        for dim, offset in enumerate((1, 2), start=1):  # H, W
   814→            length = mrope_section[dim] * 3
   815→            idx = slice(offset, length, 3)
   816→            freqs_t[..., idx] = freqs[dim, ..., idx]
   817→        return freqs_t
   818→
   819→    @torch.no_grad()
   820→    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
   821→    def forward(self, x, position_ids):
   822→        # In contrast to other models, Qwen3ASRThinker has different position ids for the grids
   823→        # So we expand the inv_freq to shape (3, ...)
   824→        if position_ids.ndim == 2:
   825→            position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
   826→        inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1)
   827→        position_ids_expanded = position_ids[:, :, None, :].float()  # shape (3, bs, 1, positions)
   828→
   829→        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
   830→        with torch.autocast(device_type=device_type, enabled=False):  # Force float32
   831→            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3)
   832→            freqs = self.apply_interleaved_mrope(freqs, self.mrope_section)
   833→            emb = torch.cat((freqs, freqs), dim=-1)
   834→            cos = emb.cos() * self.attention_scaling
   835→            sin = emb.sin() * self.attention_scaling
   836→
   837→        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
   838→
   839→
   840→class Qwen3ASRThinkerTextMLP(nn.Module):
   841→    def __init__(self, config, intermediate_size=None):
   842→        super().__init__()
   843→        self.config = config
   844→        self.hidden_size = config.hidden_size
   845→        self.intermediate_size = intermediate_size if intermediate_size is not None else config.intermediate_size
   846→        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
   847→        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
   848→        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
   849→        self.act_fn = ACT2FN[config.hidden_act]
   850→
   851→    def forward(self, x):
   852→        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
   853→        return down_proj
   854→
   855→
   856→@use_kernel_forward_from_hub("RMSNorm")
   857→class Qwen3ASRThinkerTextRMSNorm(nn.Module):
   858→    def __init__(self, hidden_size, eps=1e-6):
   859→        """
   860→        Qwen3ASRThinkerTextRMSNorm is equivalent to T5LayerNorm
   861→        """
   862→        super().__init__()
   863→        self.weight = nn.Parameter(torch.ones(hidden_size))
   864→        self.variance_epsilon = eps
   865→
   866→    def forward(self, hidden_states):
   867→        input_dtype = hidden_states.dtype
   868→        hidden_states = hidden_states.to(torch.float32)
   869→        variance = hidden_states.pow(2).mean(-1, keepdim=True)
   870→        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
   871→        return self.weight * hidden_states.to(input_dtype)
   872→
   873→    def extra_repr(self):
   874→        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
   875→
   876→
   877→class Qwen3ASRThinkerTextAttention(nn.Module):
   878→    """Multi-headed attention from 'Attention Is All You Need' paper"""
   879→
   880→    def __init__(self, config, layer_idx):
   881→        super().__init__()
   882→        self.config = config
   883→        self.layer_idx = layer_idx
   884→        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
   885→        self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
   886→        self.scaling = self.head_dim**-0.5
   887→        self.attention_dropout = config.attention_dropout
   888→        self.is_causal = True
   889→
   890→        self.q_proj = nn.Linear(
   891→            config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
   892→        )
   893→        self.k_proj = nn.Linear(
   894→            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
   895→        )
   896→        self.v_proj = nn.Linear(
   897→            config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
   898→        )
   899→        self.o_proj = nn.Linear(
   900→            config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias
   901→        )
   902→        self.q_norm = Qwen3ASRThinkerTextRMSNorm(
   903→            self.head_dim, eps=config.rms_norm_eps
   904→        )  # unlike olmo, only on the head dim!
   905→        self.k_norm = Qwen3ASRThinkerTextRMSNorm(
   906→            self.head_dim, eps=config.rms_norm_eps
   907→        )  # thus post q_norm does not need reshape
   908→        self.sliding_window = None
   909→
   910→    @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")
   911→    def forward(
   912→        self,
   913→        hidden_states: torch.Tensor,
   914→        position_embeddings: tuple[torch.Tensor, torch.Tensor],
   915→        attention_mask: Optional[torch.Tensor],
   916→        past_key_values: Optional[Cache] = None,
   917→        cache_position: Optional[torch.LongTensor] = None,
   918→        **kwargs: Unpack[FlashAttentionKwargs],
   919→    ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
   920→        input_shape = hidden_states.shape[:-1]
   921→        hidden_shape = (*input_shape, -1, self.head_dim)
   922→
   923→        query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
   924→        key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
   925→        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
   926→
   927→        cos, sin = position_embeddings
   928→        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
   929→
   930→        if past_key_values is not None:
   931→            # sin and cos are specific to RoPE models; cache_position needed for the static cache
   932→            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
   933→            key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs)
   934→
   935→        attention_interface: Callable = eager_attention_forward
   936→        if self.config._attn_implementation != "eager":
   937→            attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
   938→
   939→        attn_output, attn_weights = attention_interface(
   940→            self,
   941→            query_states,
   942→            key_states,
   943→            value_states,
   944→            attention_mask,
   945→            dropout=0.0 if not self.training else self.attention_dropout,
   946→            scaling=self.scaling,
   947→            sliding_window=self.sliding_window,  # diff with Llama
   948→            **kwargs,
   949→        )
   950→
   951→        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
   952→        attn_output = self.o_proj(attn_output)
   953→        return attn_output, attn_weights
   954→
   955→
   956→@auto_docstring(
   957→    custom_intro=(
   958→        "Text part of Qwen3ASRThinker, "
   959→    )
   960→)
   961→class Qwen3ASRThinkerTextModel(Qwen3ASRPreTrainedModel):
   962→    config: Qwen3ASRConfig
   963→    _no_split_modules = ["Qwen3ASRThinkerTextDecoderLayer"]
   964→    config_class = Qwen3ASRConfig
   965→    _can_record_outputs = {
   966→        "hidden_states": Qwen3ASRThinkerTextDecoderLayer,
   967→        "attentions": Qwen3ASRThinkerTextAttention,
   968→    }
   969→
   970→    def __init__(self, config: Qwen3ASRConfig):
   971→        super().__init__(config)
   972→        self.padding_idx = config.pad_token_id
   973→        self.vocab_size = config.vocab_size
   974→
   975→        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
   976→        self.layers = nn.ModuleList(
   977→            [Qwen3ASRThinkerTextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
   978→        )
   979→        self.norm = Qwen3ASRTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
   980→        self.rotary_emb = Qwen3ASRThinkerTextRotaryEmbedding(config)
   981→        self.gradient_checkpointing = False
   982→
   983→        # Initialize weights and apply final processing
   984→        self.post_init()
   985→
   986→    @check_model_inputs()
   987→    @auto_docstring
   988→    def forward(
   989→        self,
   990→        input_ids: Optional[torch.LongTensor] = None,
   991→        attention_mask: Optional[torch.Tensor] = None,
   992→        position_ids: Optional[torch.LongTensor] = None,
   993→        past_key_values: Optional[Cache] = None,
   994→        inputs_embeds: Optional[torch.FloatTensor] = None,
   995→        use_cache: Optional[bool] = None,
   996→        cache_position: Optional[torch.LongTensor] = None,
   997→        **kwargs: Unpack[FlashAttentionKwargs],
   998→    ) -> Union[tuple, BaseModelOutputWithPast]:
   999→        if (input_ids is None) ^ (inputs_embeds is not None):


Now let me check for the Thinker model which combines audio and text:


Bash

grep -n "class.*Thinker\|class.*ForConditionalGeneration" /home/ubuntu/training/qwen3-asr-1.7b-phase2-sft/qwen_asr/core/transformers_backend/modeling_qwen3_asr.py | head -10


236:class Qwen3ASRThinkerTextDecoderLayer(GradientCheckpointingLayer):
299:class Qwen3ASRThinkerCausalLMOutputWithPast(MoeCausalLMOutputWithPast):
320:class Qwen3ASRPreTrainedModelForConditionalGeneration(Qwen3ASRPreTrainedModel):
781:class Qwen3ASRThinkerTextRotaryEmbedding(nn.Module):
840:class Qwen3ASRThinkerTextMLP(nn.Module):
857:class Qwen3ASRThinkerTextRMSNorm(nn.Module):
877:class Qwen3ASRThinkerTextAttention(nn.Module):
961:class Qwen3ASRThinkerTextModel(Qwen3ASRPreTrainedModel):
1067:class Qwen3ASRThinkerForConditionalGeneration(Qwen3ASRPreTrainedModelForConditionalGeneration, GenerationMixin):
1293:class Qwen3ASRThinkerTextPreTrainedModel(PreTrainedModel):


Let me read the main Qwen3ASRForConditionalGeneration class:


  1067→class Qwen3ASRThinkerForConditionalGeneration(Qwen3ASRPreTrainedModelForConditionalGeneration, GenerationMixin):
  1068→    config: Qwen3ASRThinkerConfig
  1069→    base_model_prefix = "thinker"
  1070→    _tied_weights_keys = ["model.embed_tokens.weight", "lm_head.weight"]
  1071→    _no_split_modules = [
  1072→        "Qwen3ASRAudioEncoderLayer",
  1073→        "Qwen3ASRThinkerTextDecoderLayer",
  1074→    ]
  1075→    _can_record_outputs = {
  1076→        "hidden_states": Qwen3ASRThinkerTextDecoderLayer,
  1077→        "attentions": Qwen3ASRThinkerTextAttention,
  1078→    }
  1079→
  1080→    def __init__(self, config):
  1081→        super().__init__(config)
  1082→        self.audio_tower = Qwen3ASRAudioEncoder._from_config(config.audio_config)
  1083→        self.vocab_size = config.text_config.vocab_size
  1084→        self.model = Qwen3ASRThinkerTextModel._from_config(config.text_config)
  1085→        if "forced_aligner" in config.model_type:
  1086→            self.lm_head = nn.Linear(config.text_config.hidden_size, config.classify_num, bias=False)
  1087→        else:
  1088→            self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
  1089→        self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1
  1090→        self.rope_deltas = None
  1091→        self.post_init()
  1092→
  1093→    def get_input_embeddings(self):
  1094→        return self.model.get_input_embeddings()
  1095→
  1096→    def set_input_embeddings(self, value):
  1097→        self.model.set_input_embeddings(value)
  1098→
  1099→    def get_audio_features(
  1100→        self,
  1101→        input_features: torch.FloatTensor,
  1102→        feature_attention_mask: Optional[torch.LongTensor] = None,
  1103→        audio_feature_lengths: Optional[torch.LongTensor] = None,
  1104→    ):
  1105→        """
  1106→        Encodes audios into continuous embeddings that can be forwarded to the language model.
  1107→
  1108→        Args:
  1109→            input_features (`torch.FloatTensor`):
  1110→                The tensors corresponding to the input audios.
  1111→            feature_attention_mask (`torch.LongTensor`, *optional*):
  1112→                Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`:
  1113→            audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*):
  1114→                The length of feature shape of each audio in LLM.
  1115→        """
  1116→        if feature_attention_mask is not None:
  1117→            audio_feature_lengths = torch.sum(feature_attention_mask, dim=1)
  1118→        else:
  1119→            audio_feature_lengths = None
  1120→        feature_lens = audio_feature_lengths if audio_feature_lengths is not None else feature_attention_mask.sum(-1)
  1121→
  1122→        # audio encoder do not support batch inference to keep precision
  1123→        audio_features = []
  1124→        for input_feature, feature_len in zip(input_features, feature_lens):
  1125→            audio_output = self.audio_tower(
  1126→                input_feature[:, :feature_len],
  1127→                feature_lens=feature_len.unsqueeze(0),
  1128→            )
  1129→            audio_feature = audio_output.last_hidden_state
  1130→            audio_features.append(audio_feature)
  1131→        audio_features = torch.cat(audio_features, dim=0)
  1132→
  1133→        return audio_features
  1134→
  1135→    def get_placeholder_mask(
  1136→        self,
  1137→        input_ids: torch.LongTensor,
  1138→        inputs_embeds: torch.FloatTensor,
  1139→    ):
  1140→        """
  1141→        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
  1142→        equal to the length of multimodal features. If the lengths are different, an error is raised.
  1143→        """
  1144→        if input_ids is None:
  1145→            special_audio_mask = (
  1146→                inputs_embeds
  1147→                == self.get_input_embeddings()(
  1148→                    torch.tensor(self.config.audio_token_id, dtype=torch.long, device=inputs_embeds.device)
  1149→                )
  1150→            ).all(-1)
  1151→        else:
  1152→            special_audio_mask = input_ids == self.config.audio_token_id
  1153→
  1154→        special_audio_mask = special_audio_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
  1155→        return special_audio_mask
  1156→
  1157→    @can_return_tuple
  1158→    @auto_docstring
  1159→    def forward(
  1160→        self,
  1161→        input_ids=None,
  1162→        input_features=None,
  1163→        attention_mask=None,
  1164→        feature_attention_mask=None,
  1165→        audio_feature_lengths=None,
  1166→        position_ids=None,
  1167→        past_key_values=None,
  1168→        inputs_embeds=None,
  1169→        rope_deltas=None,
  1170→        labels=None,
  1171→        use_cache=None,
  1172→        cache_position=None,
  1173→        **kwargs,
  1174→    ) -> Union[tuple, Qwen3ASRThinkerCausalLMOutputWithPast]:
  1175→        r"""
  1176→        feature_attention_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`, *optional*):
  1177→            Mask to avoid performing attention on padding feature indices. Mask values selected in `[0, 1]`:
  1178→            - 1 for tokens that are **not masked**,
  1179→            - 0 for tokens that are **masked**.
  1180→        audio_feature_lengths (`torch.LongTensor` of shape `(num_audios)`, *optional*):
  1181→            The length of feature shape of each audio in LLM.
  1182→        rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):
  1183→            The rope index difference between sequence length and multimodal rope.
  1184→        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
  1185→            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
  1186→            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
  1187→            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
  1188→        """
  1189→
  1190→        if inputs_embeds is None:
  1191→            # 1. Extract the input embeddings
  1192→            inputs_embeds = self.get_input_embeddings()(input_ids)
  1193→
  1194→        # 2. Merge text, audios
  1195→        if input_features is not None:
  1196→            audio_features = self.get_audio_features(
  1197→                input_features,
  1198→                feature_attention_mask=feature_attention_mask,
  1199→                audio_feature_lengths=audio_feature_lengths,
  1200→            )
  1201→            audio_features = audio_features.to(inputs_embeds.device, inputs_embeds.dtype)
  1202→            audio_mask = self.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds)
  1203→            inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_features)
  1204→
  1205→        if feature_attention_mask is not None:
  1206→            audio_feature_lengths = torch.sum(feature_attention_mask, dim=1)
  1207→        else:
  1208→            audio_feature_lengths = None
  1209→
  1210→        if attention_mask is not None and position_ids is None:
  1211→            if (
  1212→                cache_position is None
  1213→                or (cache_position is not None and cache_position[0] == 0)
  1214→                or self.rope_deltas is None
  1215→            ):
  1216→                delta0 = (1 - attention_mask).sum(dim=-1).unsqueeze(1)
  1217→                position_ids, rope_deltas = self.get_rope_index(
  1218→                    attention_mask,
  1219→                )
  1220→                rope_deltas = rope_deltas - delta0
  1221→                self.rope_deltas = rope_deltas
  1222→            else:
  1223→                batch_size, seq_length = input_ids.shape
  1224→                delta = cache_position[0] + self.rope_deltas if cache_position is not None else 0
  1225→                position_ids = torch.arange(seq_length, device=input_ids.device)
  1226→                position_ids = position_ids.view(1, -1).expand(batch_size, -1)
  1227→                position_ids = position_ids.add(delta)
  1228→                position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
  1229→
  1230→        outputs = self.model(
  1231→            attention_mask=attention_mask,
  1232→            position_ids=position_ids,
  1233→            past_key_values=past_key_values,
  1234→            inputs_embeds=inputs_embeds,
  1235→            use_cache=use_cache,
  1236→            cache_position=cache_position,
  1237→            **kwargs,
  1238→        )
  1239→
  1240→        hidden_states = outputs[0]
  1241→        logits = self.lm_head(hidden_states)
  1242→
  1243→        loss = None
  1244→        if labels is not None:
  1245→            loss = self.loss_function(
  1246→                logits=logits, labels=labels, vocab_size=self.config.get_text_config().vocab_size
  1247→            )
  1248→
  1249→        return Qwen3ASRThinkerCausalLMOutputWithPast(
  1250→            loss=loss,
  1251→            logits=logits,
  1252→            hidden_states=outputs.hidden_states,
  1253→            attentions=outputs.attentions,
  1254→            past_key_values=outputs.past_key_values,
  1255→            rope_deltas=self.rope_deltas,
  1256→        )
  1257→
  1258→    def prepare_inputs_for_generation(
  1259→        self,
  1260→        input_ids,
  1261→        past_key_values=None,
  1262→        attention_mask=None,
  1263→        inputs_embeds=None,
  1264→        cache_position=None,
  1265→        position_ids=None,
  1266→        use_cache=True,
  1267→        input_features=None,
  1268→        feature_attention_mask=None,
  1269→        **kwargs,
  1270→    ):
  1271→        model_inputs = super().prepare_inputs_for_generation(
  1272→            input_ids,
  1273→            past_key_values=past_key_values,
  1274→            attention_mask=attention_mask,
  1275→            inputs_embeds=inputs_embeds,
  1276→            cache_position=cache_position,
  1277→            position_ids=position_ids,
  1278→            use_cache=use_cache,
  1279→            input_features=input_features,
  1280→            feature_attention_mask=feature_attention_mask,
  1281→            **kwargs,
  1282→        )
  1283→
  1284→        model_inputs["position_ids"] = None
  1285→
  1286→        if cache_position[0] != 0:
  1287→            model_inputs["input_features"] = None
  1288→
  1289→        return model_inputs
  1290→
  1291→
  1292→@auto_docstring
  1293→class Qwen3ASRThinkerTextPreTrainedModel(PreTrainedModel):
  1294→    config = Qwen3ASRConfig
  1295→    base_model_prefix = "model"
  1296→    supports_gradient_checkpointing = True
  1297→    _no_split_modules = ["Qwen3ASRThinkerTextDecoderLayer"]
  1298→    _skip_keys_device_placement = ["past_key_values"]
  1299→    _supports_flash_attn = True
  1300→    _supports_sdpa = True
  1301→    _supports_flex_attn = True
  1302→    _can_compile_fullgraph = False  # MoE models don't work with torch.compile (`torch.where(condition)` not supported)
  1303→    _supports_attention_backend = True
  1304→    _can_record_outputs = {
  1305→        "hidden_states": Qwen3ASRThinkerTextDecoderLayer,
  1306→        "attentions": Qwen3ASRThinkerTextAttention,
  1307→    }
  1308→    config_class = Qwen3ASRConfig
  1309→
  1310→
  1311→class Qwen3ASRForConditionalGeneration(Qwen3ASRPreTrainedModel, GenerationMixin):
  1312→    config_class = Qwen3ASRConfig
  1313→
  1314→    def __init__(self, config: Qwen3ASRConfig):
  1315→        super().__init__(config)
  1316→        self.config = config


Let me read a bit more of the ForConditionalGeneration class:


  1311→class Qwen3ASRForConditionalGeneration(Qwen3ASRPreTrainedModel, GenerationMixin):
  1312→    config_class = Qwen3ASRConfig
  1313→
  1314→    def __init__(self, config: Qwen3ASRConfig):
  1315→        super().__init__(config)
  1316→        self.config = config
  1317→
  1318→        self.thinker = Qwen3ASRThinkerForConditionalGeneration._from_config(config.thinker_config)
  1319→        self.post_init()
  1320→
  1321→    def get_support_languages(self):
  1322→        return self.config.support_languages
  1323→
  1324→    @torch.no_grad()
  1325→    def generate(
  1326→        self,
  1327→        input_ids: Optional[torch.Tensor] = None,
  1328→        max_new_tokens: int = 4096,
  1329→        eos_token_id: int | list[int] = [151645, 151643],
  1330→        **kwargs,
  1331→    ):
  1332→        shared_kwargs = {}
  1333→        thinker_kwargs = {
  1334→            "max_new_tokens": max_new_tokens,
  1335→            "eos_token_id": eos_token_id,
  1336→        }
  1337→
  1338→        for key, value in kwargs.items():
  1339→            # Process special input values
  1340→            if key == "feature_attention_mask":
  1341→                thinker_kwargs[key] = value
  1342→            elif key in ("input_features", "attention_mask"):
  1343→                thinker_kwargs[key] = value
  1344→            # Put other key to shared kwargs
  1345→            else:
  1346→                shared_kwargs[key] = value
  1347→
  1348→        # Merge kwargs
  1349→        for key, value in shared_kwargs.items():
  1350→            if key not in thinker_kwargs:
  1351→                thinker_kwargs[key] = value
  1352→
  1353→        thinker_result = self.thinker.generate(input_ids=input_ids, return_dict_in_generate=True, **thinker_kwargs)
  1354→
  1355→        return thinker_result
  1356→
  1357→
  1358→__all__ = [
  1359→    "Qwen3ASRForConditionalGeneration",
  1360→    "Qwen3ASRThinkerTextModel",
  1361→    "Qwen3ASRThinkerForConditionalGeneration",
  1362→    "Qwen3ASRPreTrainedModel",
  1363→    "Qwen3ASRPreTrainedModelForConditionalGeneration",
  1364→    "Qwen3ASRThinkerTextPreTrainedModel",
  1365→]
  1366→


Perfect! Now let me check for the evaluation scripts:


find /home/ubuntu/training/qwen3-asr-1.7b-phase2-sft -type f -name "*eval*" -o -name "*benchmark*" | grep -E "\.(py|sh)$"


/home/ubuntu/training/qwen3-asr-1.7b-phase2-sft/run_benchmarks_v2.sh
/home/ubuntu/training/qwen3-asr-1.7b-phase2-sft/run_benchmarks.sh


Let me check those scripts:


     1→#!/usr/bin/env bash
     2→# Benchmark v2: adjusted after BS=32 OOM
     3→set -euo pipefail
     4→
     5→export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
     6→export TORCH_NCCL_ASYNC_ERROR_HANDLING=3
     7→export OMP_NUM_THREADS=1
     8→export MKL_NUM_THREADS=1
     9→export OPENBLAS_NUM_THREADS=1
    10→export NUMEXPR_NUM_THREADS=1
    11→export NCCL_LAUNCH_MODE=PARALLEL
    12→export TORCHINDUCTOR_CACHE_DIR="/tmp/torch_inductor_cache"
    13→export PYTHONPATH="/root/data/Qwen3-ASR-official:${PYTHONPATH:-}"
    14→
    15→MODEL_PATH="/root/data/qwen3_asr_weights"
    16→TRAIN_FILE="/root/gemini-asr/lf_asr/artifacts/phase2/train.parquet"
    17→OUTPUT_DIR="/root/data/qwen3-asr-phase2-bench"
    18→LOG_DIR="/root/data/qwen3-asr-phase2-bench/logs"
    19→STEPS=100
    20→
    21→mkdir -p "$OUTPUT_DIR" "$LOG_DIR"
    22→
    23→run_experiment() {
    24→    local EXP_ID="$1"
    25→    shift
    26→
    27→    echo ""
    28→    echo "========================================"
    29→    echo "  Experiment: $EXP_ID"
    30→    echo "  Args: $@"
    31→    echo "========================================"
    32→
    33→    rm -rf "$OUTPUT_DIR/checkpoint-*" 2>/dev/null || true
    34→
    35→    # Background GPU monitor
    36→    (while true; do
    37→        nvidia-smi --query-gpu=index,memory.used,utilization.gpu,power.draw --format=csv,noheader
    38→        sleep 5
    39→    done) > "$LOG_DIR/${EXP_ID}_gpu_trace.csv" 2>/dev/null &
    40→    GPU_MON_PID=$!
    41→
    42→    START_TIME=$(date +%s)
    43→
    44→    torchrun --standalone --nproc_per_node=8 \
    45→        finetuning/qwen3_asr_sft_phase2.py \
    46→        --model_path "${MODEL_PATH}" \
    47→        --train_file "${TRAIN_FILE}" \
    48→        --output_dir "${OUTPUT_DIR}" \
    49→        --grad_acc 1 \
    50→        --lr 2e-5 \
    51→        --max_steps "${STEPS}" \
    52→        --log_steps 10 \
    53→        --save_steps 999999 \
    54→        --eval_steps 999999 \
    55→        --num_workers 8 \
    56→        --prefetch_factor 4 \
    57→        --pin_memory 1 \
    58→        --persistent_workers 1 \
    59→        --max_open_tars 32 \
    60→        --ddp_static_graph 1 \
    61→        --language_tag_mode auto \
    62→        --max_duration_s 30.0 \
    63→        --seed 42 \
    64→        "$@" \
    65→        2>&1 | tee "$LOG_DIR/${EXP_ID}.log"
    66→
    67→    RET=$?
    68→    END_TIME=$(date +%s)
    69→    ELAPSED=$((END_TIME - START_TIME))
    70→
    71→    kill $GPU_MON_PID 2>/dev/null || true
    72→
    73→    echo ""
    74→    echo "--- $EXP_ID Summary ---"
    75→    if [ $RET -ne 0 ]; then
    76→        echo "STATUS: FAILED (exit $RET)"
    77→    else
    78→        echo "STATUS: SUCCESS"
    79→        echo "Wall time: ${ELAPSED}s for $STEPS steps"
    80→        grep "'train_samples_per_second'" "$LOG_DIR/${EXP_ID}.log" | tail -1 || true
    81→        grep "'train_loss'" "$LOG_DIR/${EXP_ID}.log" | tail -1 || true
    82→    fi
    83→    if [ -f "$LOG_DIR/${EXP_ID}_gpu_trace.csv" ]; then
    84→        MAX_MEM=$(awk -F', ' '{gsub(/ MiB/,"",$2); print $2}' "$LOG_DIR/${EXP_ID}_gpu_trace.csv" | sort -n | tail -1)
    85→        AVG_UTIL=$(awk -F', ' '{gsub(/ %/,"",$3); sum+=$3; n++} END{printf "%.1f", sum/n}' "$LOG_DIR/${EXP_ID}_gpu_trace.csv")
    86→        echo "Peak memory: ${MAX_MEM} MiB | Avg GPU util: ${AVG_UTIL}%"
    87→    fi
    88→    echo "========================="
    89→    return $RET
    90→}
    91→
    92→echo "Benchmark v2 starting at $(date)"
    93→
    94→# E2: BS=16 + torch.compile reduce-overhead
    95→run_experiment "E2_bs16_compile" \
    96→    --batch_size 16 --gradient_checkpointing 0 \
    97→    --torch_compile 1 --compile_mode reduce-overhead
    98→
    99→# E3: BS=24 no compile (test if it fits with 20GB headroom)
   100→run_experiment "E3_bs24_nocompile" \
   101→    --batch_size 24 --gradient_checkpointing 0 || echo "E3 OOM"
   102→
   103→# E4: BS=32 + gradient_checkpointing ON (trade speed for larger batch)
   104→run_experiment "E4_bs32_gc" \
   105→    --batch_size 32 --gradient_checkpointing 1
   106→
   107→# E5: BS=48 + gradient_checkpointing ON
   108→run_experiment "E5_bs48_gc" \
   109→    --batch_size 48 --gradient_checkpointing 1 || echo "E5 OOM"
   110→
   111→# E6: BS=32 + gradient_checkpointing + torch.compile
   112→run_experiment "E6_bs32_gc_compile" \
   113→    --batch_size 32 --gradient_checkpointing 1 \
   114→    --torch_compile 1 --compile_mode reduce-overhead || echo "E6 FAILED"
   115→
   116→echo ""
   117→echo "All benchmarks complete at $(date)"
   118→


-rw-rw-r-- 1 ubuntu ubuntu 18555 Mar 25 02:16 /home/ubuntu/training/qwen3-asr-1.7b-phase2-sft/OVERNIGHT_TODO.md
-rw-rw-r-- 1 ubuntu ubuntu 60115 Mar 25 02:16 /home/ubuntu/training/qwen3-asr-1.7b-phase2-sft/README.md


Perfect! Now let me look at the README for additional context:


1→# Qwen3-ASR
     2→
     3→

     4→
     5→

     6→
     7→

     8→
     9→

    10→&nbsp&nbsp🤗 Hugging Face&nbsp&nbsp | &nbsp&nbsp🤖 ModelScope&nbsp&nbsp | &nbsp&nbsp📑 Blog&nbsp&nbsp | &nbsp&nbsp📑 Paper&nbsp&nbsp
    11→

    12→🖥 Hugging Face Demo&nbsp&nbsp | &nbsp&nbsp 🖥 ModelScope Demo&nbsp&nbsp | &nbsp&nbsp💬 WeChat (微信)&nbsp&nbsp | &nbsp&nbsp🫨 Discord&nbsp&nbsp | &nbsp&nbsp📑 API
    13→
    14→

    15→
    16→We release **Qwen3-ASR**, a family that includes two powerful all-in-one speech recognition models that support language identification and ASR for 52 languages and dialects, as well as a novel non-autoregressive speech forced-alignment model that can align text–speech pairs in 11 languages.
    17→
    18→
    19→## News
    20→* 2026.1.29: 🎉🎉🎉 We have released the [Qwen3-ASR](https://huggingface.co/collections/Qwen/qwen3-asr) series (0.6B/1.7B) and the Qwen3-ForcedAligner-0.6B model. Please check out our [blog](https://qwen.ai/blog?id=qwen3asr)!
    21→
    22→
    23→## Contents
    24→
    25→- [Overview](#overview)
    26→  - [Introduction](#introduction)
    27→  - [Model Architecture](#model-architecture)
    28→  - [Released Models Description and Download](#released-models-description-and-download)
    29→- [Quickstart](#quickstart)
    30→  - [Environment Setup](#environment-setup)
    31→  - [Python Package Usage](#python-package-usage)
    32→    - [Quick Inference](#quick-inference)
    33→    - [vLLM Backend](#vllm-backend)
    34→    - [Streaming Inference](#streaming-inference)
    35→    - [ForcedAligner Usage](#forcedaligner-usage)
    36→  - [DashScope API Usage](#dashscope-api-usage)
    37→- [Launch Local Web UI Demo](#launch-local-web-ui-demo)
    38→  - [Gradio Demo](#gradio-demo)
    39→  - [Streaming Demo](#streaming-demo)
    40→- [Deployment with vLLM](#deployment-with-vllm)
    41→- [Fine Tuning](#fine-tuning)
    42→- [Docker](#docker)
    43→- [Evaluation](#evaluation)
    44→- [Citation](#citation)
    45→
    46→
    47→## Overview
    48→
    49→### Introduction
    50→
    51→

    52→
    53→

    54→
    55→The Qwen3-ASR family includes Qwen3-ASR-1.7B and Qwen3-ASR-0.6B, which support language identification and ASR for 52 languages and dialects. Both leverage large-scale speech training data and the strong audio understanding capability of their foundation model, Qwen3-Omni. Experiments show that the 1.7B version achieves state-of-the-art performance among open-source ASR models and is competitive with the strongest proprietary commercial APIs. Here are the main features:
    56→
    57→* **All-in-one**: Qwen3-ASR-1.7B and Qwen3-ASR-0.6B support language identification and speech recognition for 30 languages and 22 Chinese dialects, so as to English accents from multiple countries and regions.
    58→
    59→* **Excellent and Fast**: The Qwen3-ASR family ASR models maintains high-quality and robust recognition under complex acoustic environments and challenging text patterns. Qwen3-ASR-1.7B achieves strong performance on both open-sourced and internal benchmarks. While the 0.6B version achieves accuracy-efficient trade-off, it reaches 2000 times throughput at a concurrency of 128. They both achieve streaming / offline unified inference with single model and support transcribe long audio.
    60→
    61→* **Novel and strong forced alignment Solution**: We introduce Qwen3-ForcedAligner-0.6B, which supports timestamp prediction for arbitrary units within up to 5 minutes of speech in 11 languages. Evaluations show its timestamp accuracy surpasses E2E based forced-alignment models.
    62→
    63→* **Comprehensive inference toolkit**: In addition to open-sourcing the architectures and weights of the Qwen3-ASR series, we also release a powerful, full-featured inference framework that supports vLLM-based batch inference, asynchronous serving, streaming inference, timestamp prediction, and more.
    64→
    65→### Model Architecture
    66→
    67→

    68→
    69→

    70→
    71→
    72→### Released Models Description and Download
    73→
    74→Below is an introduction and download information for the Qwen3-ASR models. Please select and download the model that fits your needs.
    75→
    76→| Model | Supported Languages | Supported Dialects | Inference Mode | Audio Types |
    77→|---|---|---|---|---|
    78→| Qwen3-ASR-1.7B & Qwen3-ASR-0.6B | Chinese (zh), English (en), Cantonese (yue), Arabic (ar), German (de), French (fr), Spanish (es), Portuguese (pt), Indonesian (id), Italian (it), Korean (ko), Russian (ru), Thai (th), Vietnamese (vi), Japanese (ja), Turkish (tr), Hindi (hi), Malay (ms), Dutch (nl), Swedish (sv), Danish (da), Finnish (fi), Polish (pl), Czech (cs), Filipino (fil), Persian (fa), Greek (el), Hungarian (hu), Macedonian (mk), Romanian (ro) | Anhui, Dongbei, Fujian, Gansu, Guizhou, Hebei, Henan, Hubei, Hunan, Jiangxi, Ningxia, Shandong, Shaanxi, Shanxi, Sichuan, Tianjin, Yunnan, Zhejiang, Cantonese (Hong Kong accent), Cantonese (Guangdong accent), Wu language, Minnan language. | Offline / Streaming | Speech, Singing Voice, Songs with BGM |
    79→| Qwen3-ForcedAligner-0.6B | Chinese, English, Cantonese, French, German, Italian, Japanese, Korean, Portuguese, Russian, Spanish | -- | NAR | Speech |
    80→
    81→During model loading in the `qwen-asr` package or vLLM, model weights will be downloaded automatically based on the model name. However, if your runtime environment does not allow downloading weights during execution, you can use the following commands to manually download the model weights to a local directory:
    82→
    83→```bash
    84→# Download through ModelScope (recommended for users in Mainland China)
    85→pip install -U modelscope
    86→modelscope download --model Qwen/Qwen3-ASR-1.7B  --local_dir ./Qwen3-ASR-1.7B
    87→modelscope download --model Qwen/Qwen3-ASR-0.6B --local_dir ./Qwen3-ASR-0.6B
    88→modelscope download --model Qwen/Qwen3-ForcedAligner-0.6B --local_dir ./Qwen3-ForcedAligner-0.6B
    89→# Download through Hugging Face
    90→pip install -U "huggingface_hub[cli]"
    91→huggingface-cli download Qwen/Qwen3-ASR-1.7B --local-dir ./Qwen3-ASR-1.7B
    92→huggingface-cli download Qwen/Qwen3-ASR-0.6B --local-dir ./Qwen3-ASR-0.6B
    93→huggingface-cli download Qwen/Qwen3-ForcedAligner-0.6B --local-dir ./Qwen3-ForcedAligner-0.6B
    94→```
    95→
    96→
    97→## Quickstart
    98→
    99→### Environment Setup
   100→
   101→The easiest way to use Qwen3-ASR is to install the `qwen-asr` Python package from PyPI. This will pull in the required runtime dependencies and allow you to load any released Qwen3-ASR model. If you’d like to simplify environment setup further, you can also use our official [Docker image](#docker). The `qwen-asr` package provides two backends: the transformers backend and the vLLM backend. For usage instructions for different backends, please refer to [Python Package Usage](#python-package-usage). We recommend using a **fresh, isolated environment** to avoid dependency conflicts with existing packages. You can create a clean Python 3.12 environment like this:
   102→
   103→```bash
   104→conda create -n qwen3-asr python=3.12 -y
   105→conda activate qwen3-asr
   106→```
   107→
   108→Run the following command to get the minimal installation with transformers-backend support:
   109→
   110→```bash
   111→pip install -U qwen-asr
   112→```
   113→
   114→To enable the vLLM backend for faster inference and streaming support, run:
   115→
   116→```bash
   117→pip install -U qwen-asr[vllm]
   118→```
   119→
   120→If you want to develop or modify the code locally, install from source in editable mode:
   121→
   122→```bash
   123→git clone https://github.com/QwenLM/Qwen3-ASR.git
   124→cd Qwen3-ASR
   125→pip install -e .
   126→# support vLLM backend
   127→# pip install -e ".[vllm]"
   128→```
   129→
   130→Additionally, we recommend using FlashAttention 2 to reduce GPU memory usage and accelerate inference speed, especially for long inputs and large batch sizes.
   131→
   132→```bash
   133→pip install -U flash-attn --no-build-isolation
   134→```
   135→
   136→If your machine has less than 96GB of RAM and lots of CPU cores, run:
   137→
   138→```bash
   139→MAX_JOBS=4 pip install -U flash-attn --no-build-isolation
   140→```
   141→
   142→Also, you should have hardware that is compatible with FlashAttention 2. Read more about it in the official documentation of the [FlashAttention repository](https://github.com/Dao-AILab/flash-attention). FlashAttention 2 can only be used when a model is loaded in `torch.float16` or `torch.bfloat16`.
   143→
   144→### Python Package Usage
   145→
   146→#### Quick Inference
   147→
   148→The `qwen-asr` package provides two backends: **transformers backend** and **vLLM backend**. You can pass audio inputs as a local path, a URL, base64 data, or a `(np.ndarray, sr)` tuple, and run batch inference. To quickly try Qwen3-ASR, you can use `Qwen3ASRModel.from_pretrained(...)` for the transformers backend with the following code:
   149→
   150→```python
   151→import torch
   152→from qwen_asr import Qwen3ASRModel
   153→
   154→model = Qwen3ASRModel.from_pretrained(
   155→    "Qwen/Qwen3-ASR-1.7B",
   156→    dtype=torch.bfloat16,
   157→    device_map="cuda:0",
   158→    # attn_implementation="flash_attention_2",
   159→    max_inference_batch_size=32, # Batch size limit for inference. -1 means unlimited. Smaller values can help avoid OOM.
   160→    max_new_tokens=256, # Maximum number of tokens to generate. Set a larger value for long audio input.
   161→)
   162→
   163→results = model.transcribe(
   164→    audio="https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav",
   165→    language=None, # set "English" to force the language
   166→)
   167→
   168→print(results[0].language)
   169→print(results[0].text)
   170→```
   171→
   172→If you want to return timestamps, pass `forced_aligner` and its init kwargs. Here is an example of batch inference with timestamps output:
   173→
   174→```python
   175→import torch
   176→from qwen_asr import Qwen3ASRModel
   177→
   178→model = Qwen3ASRModel.from_pretrained(
   179→    "Qwen/Qwen3-ASR-1.7B",
   180→    dtype=torch.bfloat16,
   181→    device_map="cuda:0",
   182→    # attn_implementation="flash_attention_2",
   183→    max_inference_batch_size=32, # Batch size limit for inference. -1 means unlimited. Smaller values can help avoid OOM.
   184→    max_new_tokens=256, # Maximum number of tokens to generate. Set a larger value for long audio input.
   185→    forced_aligner="Qwen/Qwen3-ForcedAligner-0.6B",
   186→    forced_aligner_kwargs=dict(
   187→        dtype=torch.bfloat16,
   188→        device_map="cuda:0",
   189→        # attn_implementation="flash_attention_2",
   190→    ),
   191→)
   192→
   193→results = model.transcribe(
   194→    audio=[
   195→      "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_zh.wav",
   196→      "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav",
   197→    ],
   198→    language=["Chinese", "English"], # can also be set to None for automatic language detection
   199→    return_time_stamps=True,
   200→)


Excellent! Now I have a comprehensive understanding. Let me create a final summary document by gathering the key information:

## Comprehensive Summary: Qwen3-ASR-1.7B-Phase2-SFT Architecture & Inference

Based on my thorough exploration of the `/home/ubuntu/training/qwen3-asr-1.7b-phase2-sft` repository, here's a complete technical overview:

---

### 1. MODEL ARCHITECTURE

**Overall Structure:**
- **Qwen3ASRForConditionalGeneration** is the main entry point, which wraps a **Qwen3ASRThinkerForConditionalGeneration** (the actual model doing inference/generation)
- Dual-component architecture: **Audio Encoder** + **Text Model (LLM)**

**Audio Encoder Component:**
- Class: `Qwen3ASRAudioEncoder` (defined in `/qwen_asr/core/transformers_backend/modeling_qwen3_asr.py`)
- **Input:** Mel-spectrogram features (128 mel bins by default)
- **Architecture:**
  - Conv2D layers (3 successive convolutions with GELU activations) for downsampling
  - 32 transformer encoder layers with:
    - Hidden size: 1280
    - Attention heads: 20
    - FFN dim: 5120
  - Sinusoidal positional embeddings
  - Output projection to 3584-dimensional embeddings
- **Key Parameters:**
  - `num_mel_bins`: 128
  - `d_model`: 1280
  - `encoder_layers`: 32
  - `n_window`: 100 (chunking for training)
  - `n_window_infer`: 400 (for inference)

**Text Model Component:**
- Class: `Qwen3ASRThinkerTextModel`
- Standard Qwen3 architecture:
  - Vocab size: 151,936
  - Hidden size: 4096
  - 32 transformer decoder layers
  - 32 attention heads
  - GQA (Grouped Query Attention)
  - RoPE (Rotary Position Embedding) with dynamic scaling
  - Max position embeddings: 128,000
- **LM Head:** Projects hidden states to vocabulary logits

---

### 2. INFERENCE/GENERATION WORKFLOW

**Primary Inference Class:** `Qwen3ASRModel` (in `/qwen_asr/inference/qwen3_asr.py`)

**Two Backends Supported:**
1. **Transformers Backend** (default):
   ```
   Qwen3ASRModel.from_pretrained(
       "Qwen/Qwen3-ASR-1.7B",
       dtype=torch.bfloat16,
       device_map="cuda:0",
       max_inference_batch_size=32,
       max_new_tokens=256
   )
   ```

2. **vLLM Backend** (for higher throughput):
   ```
   Qwen3ASRModel.LLM(
       "Qwen/Qwen3-ASR-1.7B",
       gpu_memory_utilization=0.8
   )
   ```

**Core Inference Steps:**

1. **Audio Normalization** (`utils.normalize_audio_input()`):
   - Converts input to mono, 16kHz, float32
   - Normalizes amplitude to [-1, 1]
   - Supports: local paths, URLs, base64 data, (np.ndarray, sr) tuples

2. **Audio Chunking** (`split_audio_into_chunks()`):
   - Splits long audio at low-energy boundaries
   - Max chunk duration: 1200s (20 min) for ASR, 180s for forced alignment
   - Guarantees exact reconstruction (no overlaps/gaps)

3. **Processor Pipeline** (`Qwen3ASRProcessor`):
   - Feature extraction: WhisperFeatureExtractor (converts waveform → mel-spectrogram)
   - Text tokenization: Qwen2TokenizerFast
   - Multimodal token handling: Replaces `<|audio|>` placeholders with correct audio token count

4. **Model Forward Pass**:
   ```
   # Audio encoding
   input_features → Qwen3ASRAudioEncoder → audio_embeddings

   # Embedding fusion
   text_embeddings = embed_tokens(input_ids)
   # Replace audio token positions with actual audio embeddings
   embeddings = masked_scatter(text_embeddings, audio_mask, audio_embeddings)

   # Text generation
   embeddings → Qwen3ASRThinkerTextModel → logits
   generate() → tokens → decode()
   ```

5. **Output Parsing** (`parse_asr_output()`):
   - Extracts language and transcription from raw output
   - Expected format: `"language [Language]<asr_text>[transcribed_text]"`
   - Handles language detection failure (defaults to empty if "language None")
   - Detects and fixes repetition artifacts

**Optional: Timestamp Alignment**:
- Uses `Qwen3ForcedAligner-0.6B` (separate NAR model)
- Aligns predicted text to audio frames
- Offsets timestamps for multi-chunk reconstruction

---

### 3. TRAINING SCRIPT STRUCTURE

**Main Script:** `/finetuning/qwen3_asr_sft_phase2.py`

**Framework:** PyTorch + Hugging Face Transformers (with custom modifications)

**Key Components:**

A. **Data Loading:**
   - `BucketedIterableDataset`: Reads from per-bucket parquet files with dynamic batch sizing
   - **Bucketing Strategy:** Same-duration batching to minimize padding
   - **Buckets:** b_0_3, b_3_5, b_5_7, b_7_10, b_10_15, b_15_20, b_20_30 (duration ranges in seconds)
   - **Per-bucket batch sizes:** Larger for short audio (BS=56 for 0-3s), smaller for long (BS=4 for 20-30s)
   - **Data Sources:** Tar files with index.json files for O(1) seek+read
   - **Worker-side decoding:** Audio decoded in DataLoader workers (parallelized)

B. **Data Collation:**
   - Class: `DataCollatorForQwen3ASRPhase2`
   - **Processing pipeline:**
     1. Extract waveform, transcript, language, prompt
     2. Audio sanitization: Check for NaN/Inf, max 30s duration
     3. Transcript cleaning: Remove markup tags
     4. Language detection: Detect script mismatch with declared language
     5. Format target: Build "language [Language]<asr_text>[transcript]" format
     6. Processor call: Convert audio→mel features + tokenize text
     7. Loss mask creation: Mark prefix tokens as -100 (ignored in loss)

C. **Training Configuration:**
   - **Trainer:** Hugging Face Trainer with custom TrainingArguments
   - **Optimization:**
     - AdamW optimizer with learning rate 2e-5
     - Gradient accumulation: configurable
     - Gradient checkpointing: optional
     - torch.compile: optional with reduce-overhead mode
   - **Loss Function:** Standard language modeling cross-entropy (only on target tokens)
   - **Callbacks:** ProfilingCallback for batch statistics, crash breadcrumbs

D. **Data Format Expected:**
   - Parquet files with columns: `sample_id`, `language`, `transcript`, `tar_path`, `tar_member_name`, `duration_s`, `bucket_id`
   - Tar files with indexed access (`.tar.index.json` files)
   - Audio encoding: WAV format in tar archives
   - Transcripts: Cleaned text without markup

---

### 4. TOKENIZER & AUDIO PROCESSOR

**Tokenizer:** `Qwen2TokenizerFast` (from Qwen2, registered for Qwen3-ASR)
- Vocab size: 151,936
- Special tokens:
  - `<|audio|>`: placeholder for audio tokens
  - `<|audio_bos|>` (151647): audio start token
  - `<|eos|>` (151645): end of sequence
- Supports chat template with `apply_chat_template()`

**Audio Feature Extractor:** `WhisperFeatureExtractor`
- Input: Mono waveform at 16kHz
- Output: Mel-spectrogram (128 mel bins)
- Sampling rate: 16000 Hz
- Returns: `input_features` and `feature_attention_mask`

**Processor:** `Qwen3ASRProcessor`
- Combines feature extractor + tokenizer
- Key method: `replace_multimodal_special_tokens()`
  - Calculates actual audio token count from mel features
  - Replaces `<|audio|>` placeholders with correct number of audio tokens
- **Audio token length calculation:**
  ```
  feat_lengths = (input_lengths % 100 - 1) // 2 + 1
  output_lengths = ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // 100) * 13
  ```

---

### 5. EVALUATION & BENCHMARKING

**Benchmark Scripts:**
- `/run_benchmarks.sh` and `/run_benchmarks_v2.sh`: Test different configurations
- Tests memory usage, throughput, training time

**Configurations Tested:**
- BS=16 with torch.compile
- BS=24 (test memory limits)
- BS=32 with gradient checkpointing
- BS=48 with gradient checkpointing
- Combined: BS=32 + GC + torch.compile

**Metrics Tracked:**
- `train_loss`, `train_samples_per_second`
- Peak GPU memory usage
- Average GPU utilization %
- Training time per batch

**No dedicated evaluation script found**, but training uses:
- Validation via `eval_steps`
- Metrics callback logging

---

### 6. DATA COLLATOR DETAILS

**File:** `/finetuning/qwen3_asr_sft_phase2.py`, class `DataCollatorForQwen3ASRPhase2`

**Input:** List of sample dicts (can be pre-batched from BucketedIterableDataset)

**Processing:**
1. **Audio Validation:**
   ```python
   # Skip NaN/Inf
   # Skip if > 30 seconds
   # Skip if decode fails (optional)
   ```

2. **Target Text Formation:**
   ```python
   target = format_target_text(
       transcript=transcript,
       language_code=language,  # detected or declared
       language_tag_mode="auto",  # or "force_text_only"
       transcript_has_prefix=False
   )
   # Result: "language [Language]<asr_text>[transcript]"
   ```

3. **Prompt Building:**
   - Uses chat template with context (optional system message)
   - Adds generation prompt to build base text

4. **Processor Call:**
   ```python
   full_inputs = self.processor(
       text=full_texts,  # prefix + target + eos
       audio=audios,     # list of waveforms
       return_tensors="pt",
       padding=True,
       truncation=False
   )
   # Returns: input_ids, attention_mask, input_features, feature_attention_mask
   ```

5. **Loss Label Masking:**
   - Calculate prefix token length (with audio token expansion)
   - Set `labels[..., :prefix_len] = -100` (ignored in loss)
   - Only target tokens contribute to loss

6. **Debug Metadata:**
   - Tracks: batch_size, audio_secs, seq_len, audio_tokens_max, label_tokens_sum
   - Returns as tensors for logging

---

### 7. KEY INFERENCE EXAMPLES

**Single Audio:**
```python
results = model.transcribe(
    audio="path/to/audio.wav",  # or URL or (np.array, sr)
    language=None,  # auto-detect
    return_time_stamps=False
)
# Returns: List[ASRTranscription]
# Each has: language (str), text (str), time_stamps (optional)
```

**Batch with Mixed Inputs:**
```python
results = model.transcribe(
    audio=[url, base64_string, (np.array, 16000)],  # mixed types
    language=["English", None, "Chinese"],  # forced or auto
    context=["some context", "", ""],  # optional
    return_time_stamps=True  # requires forced_aligner
)
```

**Streaming (vLLM backend only):**
```python
state = model.init_streaming_state(
    context="",
    language=None,
    chunk_size_sec=2.0  # 2s chunks
)

for audio_chunk in audio_stream:
    state = model.process_streaming_chunk(state, audio_chunk)
    print(state.text)  # incremental output
```

---

### 8. FILE STRUCTURE

```
/qwen_asr/
├── core/
│   ├── transformers_backend/
│   │   ├── configuration_qwen3_asr.py    # Config classes
│   │   ├── modeling_qwen3_asr.py         # Model architecture
│   │   ├── processing_qwen3_asr.py       # Processor (audio+text)
│   │   └── __init__.py                   # Exports
│   └── vllm_backend/
│       ├── qwen3_asr.py                  # vLLM integration
│       └── __init__.py
├── inference/
│   ├── qwen3_asr.py                      # Main Qwen3ASRModel class
│   ├── qwen3_forced_aligner.py           # Forced alignment
│   └── utils.py                          # Audio I/O, parsing
├── cli/
│   ├── demo.py                           # Gradio web UI
│   ├── demo_streaming.py                 # Streaming demo
│   └── serve.py                          # Server setup
└── __main__.py

/finetuning/
├── qwen3_asr_sft.py                      # Phase 1 training script
└── qwen3_asr_sft_phase2.py               # Phase 2 training script

/examples/
├── example_qwen3_asr_transformers.py     # Transformers backend example
├── example_qwen3_asr_vllm.py             # vLLM backend example
└── example_qwen3_forced_aligner.py       # Forced alignment example
```

---

### 9. KEY TAKEAWAYS FOR INFERENCE

To generate transcriptions from audio files:

1. **Load Model:**
   ```python
   model = Qwen3ASRModel.from_pretrained("Qwen/Qwen3-ASR-1.7B", device_map="cuda:0")
   ```

2. **Call Transcribe:**
   ```python
   results = model.transcribe(audio="path/to/audio.wav")
   ```

3. **Extract Results:**
   ```python
   for r in results:
       print(f"Language: {r.language}")
       print(f"Text: {r.text}")
       if r.time_stamps:  # if enabled
           for ts in r.time_stamps:
               print(f"  {ts.text}: {ts.start_time}-{ts.end_time}s")
   ```

The model handles all preprocessing internally: audio loading, resampling, mel-spectrogram extraction, tokenization, and decoding. The processor seamlessly integrates audio and text modalities by tokenizing them separately and using special placeholder tokens to mark audio positions.