# coding=utf-8 # Adapted from # https://github.com/huggingface/transformers/blob/19e6e80e10118f855137b90740936c0b11ac397f/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py # Copyright 2024 The Qwen team. # Copyright 2023 The vLLM team. # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. # # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX # and OPT implementations in this library. It has been modified from its # original forms to accommodate minor architectural differences compared # to GPT-NeoX and OPT used by the Meta AI team that trained the model. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Inference-only Qwen2-VL model compatible with HuggingFace weights.""" import logging import re from functools import partial from typing import Iterable, List, Optional, Tuple, Type import torch import torch.nn as nn import torch.nn.functional as F from einops import rearrange from transformers.activations import ACT2FN from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import ( Qwen2_5_VLConfig, Qwen2_5_VLVisionConfig, ) from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import ( Qwen2_5_VisionPatchEmbed, Qwen2_5_VisionRotaryEmbedding, ) from sglang.srt.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) from sglang.srt.distributed.parallel_state import get_pp_group from sglang.srt.environ import envs from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, MergedColumnParallelLinear, RowParallelLinear, ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.pooler import Pooler, PoolingType from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import PPMissingLayer, get_layer_id from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.managers.mm_utils import ( MultiModalityDataPaddingPatternMultimodalTokens, general_mm_embed_routine, ) from sglang.srt.managers.schedule_batch import ( Modality, MultimodalDataItem, MultimodalInputs, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.qwen2 import Qwen2Model from sglang.srt.models.utils import RotaryPosMixin, WeightsMapper, permute_inv from sglang.srt.multimodal.mm_utils import run_dp_sharded_mrope_vision_model from sglang.srt.multimodal.vit_cuda_graph_runner import ViTCudaGraphRunner from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix, is_cuda, is_npu _is_cuda = is_cuda() logger = logging.getLogger(__name__) class Qwen2_5_VLMLP(nn.Module): def __init__( self, in_features: int, hidden_features: int = None, bias: bool = True, hidden_act="silu", quant_config: Optional[QuantizationConfig] = None, prefix: str = "", use_data_parallel: bool = False, ): super().__init__() self.tp_size = ( 1 if use_data_parallel else get_tensor_model_parallel_world_size() ) self.tp_rank = 0 if use_data_parallel else get_tensor_model_parallel_rank() self.gate_up_proj = MergedColumnParallelLinear( input_size=in_features, output_sizes=[hidden_features] * 2, # [gate_proj, up_proj] bias=bias, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), tp_size=self.tp_size, tp_rank=self.tp_rank, ) self.down_proj = RowParallelLinear( hidden_features, in_features, bias=bias, quant_config=quant_config, prefix=add_prefix("down_proj", prefix), tp_size=self.tp_size, tp_rank=self.tp_rank, ) self.hidden_act = hidden_act if self.hidden_act == "silu": self.act = SiluAndMul() else: base_act = ACT2FN[self.hidden_act] def _act_fn(x: torch.Tensor) -> torch.Tensor: gate, up = x.chunk(2, dim=-1) return base_act(gate) * up self.act = _act_fn def forward(self, x: torch.Tensor) -> torch.Tensor: gate_up, _ = self.gate_up_proj(x) x = self.act(gate_up) x_down, _ = self.down_proj(x) return x_down class Qwen2_5_VisionBlock(nn.Module): def __init__( self, dim: int, intermediate_dim: int, num_heads: int, hidden_act="silu", norm_layer: Type[nn.Module] = None, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", num_dummy_heads: int = 0, rms_norm_eps: float = 1e-6, use_data_parallel: bool = False, ) -> None: super().__init__() self.norm1 = RMSNorm(dim, eps=rms_norm_eps) self.norm2 = RMSNorm(dim, eps=rms_norm_eps) self.attn = VisionAttention( embed_dim=dim, num_heads=num_heads, projection_size=dim, use_qkv_parallel=True, proj_bias=True, flatten_batch=True, quant_config=quant_config, prefix=add_prefix("attn", prefix), num_dummy_heads=num_dummy_heads, use_data_parallel=use_data_parallel, ) self.mlp = Qwen2_5_VLMLP( dim, intermediate_dim, hidden_act=hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), use_data_parallel=use_data_parallel, ) def forward( self, x: torch.Tensor, cu_seqlens: torch.Tensor, position_embeddings: torch.Tensor, output_ws=None, ) -> torch.Tensor: S, B, H = x.shape # norm1: flatten to 2D -> [S*B, H], then reshape back x2d = x.reshape(-1, H) hidden_states = self.norm1(x2d).reshape(S, B, H) # Attention expects [B, S, H] hidden_states = rearrange(hidden_states, "s b h -> b s h") attn = self.attn( hidden_states, cu_seqlens=cu_seqlens, position_embeddings=position_embeddings, output_ws=output_ws, ) attn = rearrange(attn, "b s h -> s b h") # norm2 with fused residual-add: also 2D attn2d = attn.reshape(-1, H) x_norm_2d, x_after_add_2d = self.norm2(x2d, residual=attn2d) x_norm = x_norm_2d.reshape(S, B, H) x_after_add = x_after_add_2d.reshape(S, B, H) # MLP and final residual mlp_out = self.mlp(x_norm) x = x_after_add + mlp_out return x class Qwen2_5_VisionPatchMerger(nn.Module): def __init__( self, dim: int, context_dim: int, spatial_merge_size: int = 2, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", use_data_parallel: bool = False, ) -> None: super().__init__() self.hidden_size = context_dim * (spatial_merge_size**2) self.ln_q = RMSNorm(context_dim, eps=1e-6) tp_size = 1 if use_data_parallel else get_tensor_model_parallel_world_size() tp_rank = 0 if use_data_parallel else get_tensor_model_parallel_rank() self.mlp = nn.ModuleList( [ ColumnParallelLinear( self.hidden_size, self.hidden_size, bias=True, quant_config=quant_config, prefix=add_prefix("mlp.0", prefix), tp_size=tp_size, tp_rank=tp_rank, ), nn.GELU(), RowParallelLinear( self.hidden_size, dim, bias=True, quant_config=quant_config, prefix=add_prefix("mlp.2", prefix), tp_size=tp_size, tp_rank=tp_rank, ), ] ) def forward(self, x: torch.Tensor) -> torch.Tensor: # x expected shape: [S, B, context_dim] S, B, D = x.shape x2d = x.reshape(-1, D) x2d = self.ln_q(x2d) # RMSNorm expects 2D x2d = x2d.view(-1, self.hidden_size) # group into spatial_merge_unit mlp_fc1, mlp_act, mlp_fc2 = self.mlp x_parallel, _ = mlp_fc1(x2d) x_parallel = mlp_act(x_parallel) out, _ = mlp_fc2(x_parallel) return out class Qwen2_5_VisionTransformer(nn.Module, RotaryPosMixin): def __init__( self, vision_config: Qwen2_5_VLVisionConfig, norm_eps: float = 1e-6, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", use_data_parallel: bool = False, max_context_len: Optional[int] = None, ) -> None: super().__init__() patch_size: int = vision_config.patch_size temporal_patch_size: int = vision_config.temporal_patch_size spatial_merge_size: int = vision_config.spatial_merge_size self.spatial_merge_size = spatial_merge_size self.spatial_merge_unit: int = spatial_merge_size * spatial_merge_size in_channels: int = vision_config.in_channels hidden_size: int = vision_config.hidden_size depth: int = vision_config.depth num_heads: int = vision_config.num_heads self.fullatt_block_indexes = vision_config.fullatt_block_indexes self.window_size = vision_config.window_size self.patch_size = vision_config.patch_size mlp_hidden_size: int = ((vision_config.intermediate_size + 7) // 8) * 8 self.use_data_parallel = use_data_parallel self.out_hidden_size = vision_config.out_hidden_size self.patch_embed = Qwen2_5_VisionPatchEmbed( patch_size=patch_size, temporal_patch_size=temporal_patch_size, in_channels=in_channels, embed_dim=hidden_size, ) norm_layer = partial(nn.LayerNorm, eps=norm_eps) head_dim = hidden_size // num_heads self.rotary_pos_emb = Qwen2_5_VisionRotaryEmbedding(head_dim // 2) self.blocks = nn.ModuleList( [ Qwen2_5_VisionBlock( dim=hidden_size, intermediate_dim=mlp_hidden_size, num_heads=num_heads, hidden_act=vision_config.hidden_act, norm_layer=norm_layer, quant_config=quant_config, prefix=add_prefix(f"blocks.{i}", prefix), use_data_parallel=use_data_parallel, ) for i in range(depth) ] ) self.merger = Qwen2_5_VisionPatchMerger( dim=vision_config.out_hidden_size, context_dim=hidden_size, spatial_merge_size=spatial_merge_size, quant_config=quant_config, prefix=add_prefix("merger", prefix), use_data_parallel=use_data_parallel, ) # Resource prepared for vit cuda graph self.tp_size = ( 1 if use_data_parallel else get_tensor_model_parallel_world_size() ) self.max_context_len = max_context_len self.enable_cg = _is_cuda and envs.SGLANG_VIT_ENABLE_CUDA_GRAPH.get() self.cuda_graph_runner: Optional[ViTCudaGraphRunner] = None if self.enable_cg: self.cuda_graph_runner = ViTCudaGraphRunner(self) def get_window_index(self, grid_thw): cu_window_seqlens: list = [0] window_index_id = 0 vit_merger_window_size = ( self.window_size // self.spatial_merge_size // self.patch_size ) window_index: list = [] for grid_t, grid_h, grid_w in grid_thw: llm_grid_h, llm_grid_w = ( grid_h // self.spatial_merge_size, grid_w // self.spatial_merge_size, ) index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape( grid_t, llm_grid_h, llm_grid_w ) pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100) index_padded = index_padded.reshape( grid_t, num_windows_h, vit_merger_window_size, num_windows_w, vit_merger_window_size, ) index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape( grid_t, num_windows_h * num_windows_w, vit_merger_window_size, vit_merger_window_size, ) seqlens = (index_padded != -100).sum([2, 3]).reshape(-1) index_padded = index_padded.reshape(-1) index_new = index_padded[index_padded != -100] window_index.append(index_new + window_index_id) cu_seqlens_tmp = ( seqlens.cumsum(0) * self.spatial_merge_unit + cu_window_seqlens[-1] ) cu_window_seqlens.extend(cu_seqlens_tmp.tolist()) window_index_id += (grid_t * llm_grid_h * llm_grid_w).item() window_index = torch.cat(window_index, dim=0) return window_index, cu_window_seqlens @property def dtype(self) -> torch.dtype: return self.patch_embed.proj.weight.dtype @property def device(self) -> torch.device: return self.patch_embed.proj.weight.device def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor: pos_ids = [] for t, h, w in grid_thw: base = self.rot_pos_ids(h, w, self.spatial_merge_size) pos_ids.append(base if t == 1 else base.repeat(t, 1)) pos_ids = torch.cat(pos_ids, dim=0) max_grid_size = grid_thw[:, 1:].max() rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) return rotary_pos_emb def forward( self, x: torch.Tensor, grid_thw: torch.Tensor, ) -> torch.Tensor: if self.enable_cg: return self.forward_with_cuda_graph(x, grid_thw) # patchify x = x.to(device=self.device, dtype=self.dtype) x = self.patch_embed(x) # compute position embedding rotary_pos_emb = self.rot_pos_emb(grid_thw) window_index, cu_window_seqlens = self.get_window_index(grid_thw) cu_window_seqlens = torch.tensor( cu_window_seqlens, device=x.device, dtype=torch.int32, ) cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens) # Move window_index to the same device as x before using it to index x window_index = window_index.to(device=x.device) reverse_indices = permute_inv(window_index) # Ensure rotary_pos_emb is on the same device/dtype as x rotary_pos_emb = rotary_pos_emb.to(device=x.device, dtype=x.dtype) seq_len, _ = x.size() x = x.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1) x = x[window_index, :, :] x = x.reshape(seq_len, -1) rotary_pos_emb = rotary_pos_emb.reshape( seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1 ) rotary_pos_emb = rotary_pos_emb[window_index, :, :] rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1) emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) position_embeddings = (emb.cos(), emb.sin()) # After building position_embeddings, make sure both cos and sin are on the same device/dtype as the attention input position_embeddings = ( position_embeddings[0].to(x.device, x.dtype), position_embeddings[1].to(x.device, x.dtype), ) # compute cu_seqlens - move cu_seqlens to GPU and make it int32 cu_seqlens = torch.cat( [ torch.tensor([0], device=x.device, dtype=torch.int32), (grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]) .cumsum(dim=0) .to(device=x.device, dtype=torch.int32), ] ) cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens]) # cu_seqlens must be on cpu because of npu_flash_attention_unpad operator restriction if is_npu(): cu_seqlens = cu_seqlens.to("cpu") # transformers x = x.unsqueeze(1) for layer_num, blk in enumerate(self.blocks): fullatt_indexes = self.fullatt_block_indexes if isinstance(fullatt_indexes, torch.Tensor): fullatt_indexes = fullatt_indexes.tolist() if layer_num in fullatt_indexes: cu_seqlens_now = cu_seqlens else: cu_seqlens_now = cu_window_seqlens x = blk( x, cu_seqlens=cu_seqlens_now, position_embeddings=position_embeddings ) # adapter x = self.merger(x) x = x[reverse_indices, :] return x def forward_with_cuda_graph( self, x: torch.Tensor, grid_thw: torch.Tensor, ) -> torch.Tensor: # patchify x = x.to(device=self.device, dtype=self.dtype) x = self.patch_embed(x) # compute position embedding rotary_pos_emb = self.rot_pos_emb(grid_thw) window_index, cu_window_seqlens = self.get_window_index(grid_thw) cu_window_seqlens = torch.tensor( cu_window_seqlens, device=x.device, dtype=torch.int32, ) cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens) window_index = window_index.to(device=x.device) reverse_indices = permute_inv(window_index) rotary_pos_emb = rotary_pos_emb.to(device=x.device, dtype=x.dtype) # patch token num seq_len, _ = x.size() # [G, M, hidden] x = x.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1) x = x[window_index, :, :] # [G, M, hidden] x = x.reshape(seq_len, -1) # [seq_len, hidden] rotary_pos_emb = rotary_pos_emb.reshape( seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1 ) rotary_pos_emb = rotary_pos_emb[window_index, :, :] rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1) emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) position_embeddings = (emb.cos(), emb.sin()) # After building position_embeddings, make sure both cos and sin are on # the same device/dtype as the attention input position_embeddings = ( position_embeddings[0].to(x.device, x.dtype), position_embeddings[1].to(x.device, x.dtype), ) # compute cu_seqlens - move cu_seqlens to GPU and make it int32 cu_seqlens = torch.cat( [ torch.tensor([0], device=x.device, dtype=torch.int32), (grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]) .cumsum(dim=0) .to(device=x.device, dtype=torch.int32), ] ) cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens]) return self.cuda_graph_runner.run( x=x, position_embeddings=position_embeddings, cu_seqlens=cu_seqlens, cu_window_seqlens=cu_window_seqlens, output_indices=reverse_indices, ) class Qwen2_5_VLForConditionalGeneration(nn.Module): # BitandBytes specific attributes default_bitsandbytes_target_modules = [ ".gate_up_proj.", ".down_proj.", ".q_proj.", ".k_proj.", ".v_proj.", ".o_proj.", ] bitsandbytes_stacked_params_mapping = { # shard_name, weight_name, index "q_proj": ("qkv_proj", 0), "k_proj": ("qkv_proj", 1), "v_proj": ("qkv_proj", 2), "gate_proj": ("gate_up_proj", 0), "up_proj": ("gate_up_proj", 1), } packed_modules_mapping = { "gate_up_proj": ["gate_proj", "up_proj"], } # To ensure correct weight loading and mapping. hf_to_sglang_mapper = WeightsMapper( orig_to_new_substr={ "attn.qkv": "attn.qkv_proj", }, orig_to_new_prefix={ # mapping for new names in checkpoint saved after transformers v4.52 "model.language_model.": "language_model.model.", "model.visual.": "visual.", # mapping for original checkpoint "lm_head.": "language_model.lm_head.", "model.": "language_model.model.", }, ) def __init__( self, config: Qwen2_5_VLConfig, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", ) -> None: super().__init__() self.pp_group = get_pp_group() self.config = config self.use_data_parallel = get_global_server_args().mm_enable_dp_encoder if not self.config.encoder_only: self.model = Qwen2Model( config, quant_config, prefix=add_prefix("model", prefix), ) if self.pp_group.is_last_rank: if self.pp_group.world_size == 1 and self.config.tie_word_embeddings: self.lm_head = self.model.embed_tokens else: self.lm_head = ParallelLMHead( self.config.vocab_size, self.config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), ) else: # ranks other than the last rank will have a placeholder layer self.lm_head = PPMissingLayer() else: # encoder_only mode: no language model, so no lm_head needed self.lm_head = None self.visual = Qwen2_5_VisionTransformer( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), # NOTE: Qwen2_5-VL vision encoder currently supports BitsAndBytes 4-bit quantization. # Other quantization methods (e.g., GPTQ, AWQ) are untested and may not be supported. quant_config=quant_config, prefix=add_prefix("visual", prefix), use_data_parallel=self.use_data_parallel, max_context_len=self.config.max_position_embeddings, ) self.is_mrope_enabled = "mrope_section" in self.config.rope_scaling self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) # For EAGLE3 support self.capture_aux_hidden_states = False def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs): pattern = MultiModalityDataPaddingPatternMultimodalTokens() return pattern.pad_input_tokens(input_ids, mm_inputs) def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: # in qwen-vl, last dim is the same pixel_values = torch.cat([item.feature for item in items], dim=0).type( self.visual.dtype ) image_grid_thw = torch.concat([item.image_grid_thw for item in items], dim=0) expected_dim = getattr(self.visual, "embed_dim", -1) if expected_dim == -1: vision_conf = self.config.vision_config expected_dim = getattr( vision_conf, "embed_dim", getattr(vision_conf, "hidden_size", -1) ) raw_patch_dim = 1176 if pixel_values.dim() == 2: current_dim = pixel_values.shape[-1] if current_dim == expected_dim: return pixel_values if current_dim != raw_patch_dim: return pixel_values assert pixel_values.dim() == 2, pixel_values.dim() assert image_grid_thw.dim() == 2, image_grid_thw.dim() if self.use_data_parallel: return run_dp_sharded_mrope_vision_model( self.visual, pixel_values, image_grid_thw.tolist(), rope_type="rope_3d" ) else: image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw) return image_embeds _lora_pattern = re.compile( r"^model\.layers\.(\d+)\.(?:self_attn|mlp)\.(?:qkv_proj|o_proj|down_proj|gate_up_proj)$" ) def should_apply_lora(self, module_name: str) -> bool: return bool(self._lora_pattern.match(module_name)) def get_video_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: # in qwen-vl, last dim is the same pixel_values = torch.cat([item.feature for item in items], dim=0).type( self.visual.dtype ) video_grid_thw = torch.concat([item.video_grid_thw for item in items], dim=0) assert pixel_values.dim() == 2, pixel_values.dim() assert video_grid_thw.dim() == 2, video_grid_thw.dim() if self.use_data_parallel: return run_dp_sharded_mrope_vision_model( self.visual, pixel_values, video_grid_thw.tolist(), rope_type="rope_3d" ) else: video_embeds = self.visual(pixel_values, grid_thw=video_grid_thw) return video_embeds def post_process( self, inputs_embeds, modalities: List[Modality], embeddings: List[torch.Tensor], indices: List[torch.Tensor], forward_batch: ForwardBatch, ) -> torch.Tensor: # Placeholder for post_process new_embeddings = [] for i, (modality, embedding, index) in enumerate( zip(modalities, embeddings, indices) ): if embedding is None or index is None: continue new_embeddings.append(embedding) return new_embeddings, forward_batch def get_input_embeddings(self): return self.model.embed_tokens @torch.no_grad() def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, forward_batch: ForwardBatch, input_embeds=None, get_embedding: bool = False, pp_proxy_tensors: Optional[PPProxyTensors] = None, ): """Run forward pass for Qwen2_5-VL. Args: input_ids: Flattened (concatenated) input_ids corresponding to a batch. positions: Flattened (concatenated) position ids corresponding to a batch. **NOTE**: If mrope is enabled (default setting for Qwen2-VL opensource models), the shape will be `(3, seq_len)`, otherwise it will be `(seq_len,). (Use input_metadata.mrope_positions to replace it) """ if self.is_mrope_enabled: positions = forward_batch.mrope_positions if not ( forward_batch.forward_mode.is_decode() or not forward_batch.contains_image_inputs() ): if self.is_mrope_enabled: assert positions.ndim == 2 and positions.size(0) == 3, ( "multimodal section rotary embedding requires " f"(3, seq_len) positions, but got {positions.size()}" ) hidden_states = general_mm_embed_routine( input_ids=input_ids, forward_batch=forward_batch, language_model=self.model, multimodal_model=self, positions=positions, pp_proxy_tensors=pp_proxy_tensors, ) aux_hidden_states = None if self.capture_aux_hidden_states: hidden_states, aux_hidden_states = hidden_states if self.pp_group.is_last_rank: if not get_embedding: return self.logits_processor( input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states, ) else: return self.pooler(hidden_states, forward_batch) else: return hidden_states def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): stacked_params_mapping = [ # (param_name, shard_name, shard_id) (".qkv_proj", ".q_proj", "q"), (".qkv_proj", ".k_proj", "k"), (".qkv_proj", ".v_proj", "v"), ("gate_up_proj", "up_proj", 1), ("gate_up_proj", "gate_proj", 0), ] params_dict = dict(self.named_parameters(remove_duplicate=False)) for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: continue if ( self.config.tie_word_embeddings and self.pp_group.is_last_rank and "model.embed_tokens.weight" in name ): if "lm_head.weight" in params_dict: lm_head_param = params_dict["lm_head.weight"] weight_loader = getattr( lm_head_param, "weight_loader", default_weight_loader ) weight_loader(lm_head_param, loaded_weight) for param_name, weight_name, shard_id in stacked_params_mapping: if weight_name not in name: continue if ( "visual" in name and "up_proj" not in name and "gate_proj" not in name ): continue name = name.replace(weight_name, param_name) layer_id = get_layer_id(name) if ( layer_id is not None and hasattr(self, "model") and hasattr(self.model, "start_layer") and ( layer_id < self.model.start_layer or layer_id >= self.model.end_layer ) ): continue # Skip loading extra bias for GPTQ models. if name.endswith(".bias") and name not in params_dict: continue # Skip loading visual/language model weights if ( self.config.encoder_only or self.config.language_only ) and name not in params_dict: continue param = params_dict[name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id) break else: if "visual" in name: # adapt to VisionAttention name = name.replace(r"attn.qkv.", r"attn.qkv_proj.") try: # Skip loading extra bias for GPTQ models. if name.endswith(".bias") and name not in params_dict: continue if name in params_dict.keys(): param = params_dict[name] else: continue except KeyError: print(params_dict.keys()) raise weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader(param, loaded_weight) def get_embed_and_head(self): return self.model.embed_tokens.weight, self.lm_head.weight def set_eagle3_layers_to_capture(self, layer_ids: Optional[List[int]] = None): self.capture_aux_hidden_states = True self.model.capture_aux_hidden_states = True if layer_ids is None: num_layers = self.config.num_hidden_layers self.model.layers_to_capture = [ 2, num_layers // 2, num_layers - 3, ] # Specific layers for EAGLE3 support else: self.model.layers_to_capture = [val + 1 for val in layer_ids] EntryClass = [Qwen2_5_VLForConditionalGeneration]