o ۾i< @sdZddlmZmZmZmZmZddlmZm Z ddl m Z ddl m Z ddlZddlZddlmZddlmmZddlmZddlmZdd lmZdd lmZmZdd l m!Z!m"Z"dd l#mZ$dd l%m&Z&dd l'm(Z(ddl)m*Z*ddl+m,Z,m-Z-ddl.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7m8Z8ddl9m:Z:ddl;mZ>ddl?m@Z@ddlAmBZBddlCmDZDddlEmFZFddlGmHZHmIZImJZJmKZKddlLmMZMmNZNmOZOmPZPmQZQmRZRmSZSddlTmUZUmVZVddlWmXZXmYZYmZZZm[Z[m\Z\dd l]m^Z^dd!l_m`Z`dd"lambZbdd#lcmdZdd$d%lemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZmd$d&lnmoZompZpmqZqmrZrmsZsmtZtmuZud$d'lvmwZwmxZxmyZyd$d(lzm{Z{m|Z|d$d)l}m~Z~mZmZmZmZd$d*lmZmZmZe1eZd+ZGd,d-d-ejZGd.d/d/ejZGd0d1d1ejZGd2d3d3ejZGd4d5d5ejZGd6d7d7exZGd8d9d9eXeZGd:d;d;eYeZe(ddGd?d@d@e|ZGdAdBdBe{ZeFjeeedCGdDdEdEejejeheleiegek ZdS)FzAInference-only Qwen3VL model compatible with HuggingFace weights.)CallableIterableIteratorMappingSequence) lru_cachepartial)islice)AnyN) BatchFeature)Qwen2VLImageProcessorFast) smart_resize)Qwen3VLProcessorQwen3VLVideoProcessor) Qwen3VLConfigQwen3VLVisionConfig) VideoMetadata)support_torch_compile) VllmConfig)BaseDummyOptionsVideoDummyOptions) get_pp_group) init_logger)_ACTIVATION_REGISTRY) Conv3dLayer)ColumnParallelLinearRowParallelLinear)LogitsProcessor)QuantizationConfig)get_rope)ParallelLMHead)default_weight_loader)MultiModelKeys)MULTIMODAL_REGISTRY)compute_mrope_for_mediacompute_retained_tokens_countcompute_retention_maskrecompute_mrope_positions)MultiModalDataDictMultiModalFeatureSpecMultiModalFieldConfigMultiModalKwargsItemMultiModalKwargsItemsPlaceholderRange VideoItem) ImageSizeMultiModalDataItems)BaseDummyInputsBuilderBaseMultiModalProcessorPromptReplacement PromptUpdatePromptUpdateDetails)IntermediateTensors) is_list_of)round_up)AttentionBackendEnum)MultiModalEmbeddingsSupportsEagle3 SupportsLoRA SupportsMRoPESupportsMultiModalSupportsMultiModalPruning SupportsPP_require_is_multimodal)Qwen2_5_VisionAttentionQwen2_5_VLImageEmbeddingInputsQwen2_5_VLImageInputsQwen2_5_VLImagePixelInputsQwen2_5_VLVideoEmbeddingInputsQwen2_5_VLVideoInputsQwen2_5_VLVideoPixelInputs)Qwen2VLMultiModalDataParserQwen2VLProcessingInfo_create_qwen2vl_field_factory)Qwen3ForCausalLM Qwen3Model)AutoWeightsLoaderPPMissingLayer WeightsMapper_merge_multimodal_embeddings maybe_prefix)get_vit_attn_backendis_vit_use_data_parallel!run_dp_sharded_mrope_vision_modelic sReZdZ    ddedededed d f fd d Zd ejd ejfddZZS)Qwen3_VisionPatchEmbed patch_sizetemporal_patch_size in_channels hidden_sizereturnNcs>t||_||_||_|||f}t||||dd|_dS)NT) kernel_sizestridebias)super__init__r\r]r_rproj)selfr\r]r^r_ra __class__W/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/qwen3_vl.pyres   zQwen3_VisionPatchEmbed.__init__xcCs:|j\}}||d|j|j|j}||||j}|SN)shapeviewr]r\rfr_)rgrlLCrjrjrkforwards zQwen3_VisionPatchEmbed.forward)rXrYrZr[) __name__ __module__ __qualname__intretorchTensorrs __classcell__rjrjrhrkrWs"rWc sdeZdZdejddfdedededeej gej fde dBd e f fd d Z d ej fd dZ ZS)Qwen3_VisionMLPFN in_featureshidden_featuresrcact_fn quant_configprefixc sVtt}t||||d|d|d|_t||||d|d|d|_||_dS)NF .linear_fc1)rcr return_biasr disable_tp .linear_fc2)rdrerUr linear_fc1r linear_fc2r)rgr}r~rcrrruse_data_parallelrhrjrkres*   zQwen3_VisionMLP.__init__rlcCs||||}|SN)rrr)rgrl mlp_outputrjrjrkrsszQwen3_VisionMLP.forward)rtrurvFsilurwboolrrxryrstrrersrzrjrjrhrkr{s&r{cseZdZejdddfdedededeejgejfdeege j fdBde dBd e d dffd d Z d ejdejdejdejdejd ejf ddZZS)Qwen3_VisionBlockNr|dim num_headsmlp_hidden_dimr norm_layerrrr`csnt|durttjdd}|||_|||_t|||||dd|_t |||d||dd|_ dS)Nư>epsz.attn) embed_dimrprojection_sizerrTz.mlp)rrcrr) rdrernn LayerNormnorm1norm2rCattnr{mlp)rgrrrrrrrrhrjrkres(    zQwen3_VisionBlock.__init__rl cu_seqlensrotary_pos_emb_cosrotary_pos_emb_sin max_seqlencCs6||j||||||d}||||}|S)Nrrrr)rrrr)rgrlrrrrrjrjrkrsszQwen3_VisionBlock.forward)rtrurvrrrwrrxryrModulerrrersrzrjrjrhrkrsF rcsteZdZ     ddededeegejfdBded ed edBd e d dffd d Z de j d e j fddZ ZS)Qwen3_VisionPatchMergerNrYFr|d_model context_dimrspatial_merge_sizeuse_postshuffle_normrrr`c stt}||d|_||_|jr|j}|dur#ttjdd}|||_t |j|jd||d|d|_ t |_ t |j|d||d|d|_dS)NrYrrTr)rcrrrr)rdrerUr_rrrrnormrrGELUrrr) rgrrrrrrrrrhrjrkres4    z Qwen3_VisionPatchMerger.__init__rlcCsZ|jr||d|j}n ||d|j}||\}}||}||\}}|Srm)rrrpr_rrr)rgrl x_parallel_outrjrjrkrs,s zQwen3_VisionPatchMerger.forward)NrYFNr|)rtrurvrwrrrrrrrerxryrsrzrjrjrhrkrs0 'rc s$eZdZ   d$dedededBdeddf fd d Zede j fd d Z ede j fd dZ e e dddededede jfddZdeeefddZdeeede jfddZde jde jfddZde jde jeeeBde jfdd Zd!eeee jfdeefd"d#ZZS)%Qwen3_VisionTransformerrNr| vision_confignorm_epsrrr`cstj_j_j_j_j_jd_j_t dr+j ng_ t jd_ j dtj _ tjjjjd_tjj_ttj|djj}t|ddd did _tj jjd d _tfd dttj D_t|t d_!j!t"j#t"j$t"j%hvrt&dj!dtfddtj'D_(dS)NrYdeepstack_visual_indexesg?r:)r\r]r^r_ri Tpartial_rotary_factor) head_size max_position is_neox_stylerope_parametersz.merger)rrrrrrc s2g|]}tjjjdd|dqS)Tz.deepstack_merger_list.)rrrrrrr)rout_hidden_sizer_r.0 layer_idxrrrrgrrjrk qs  z4Qwen3_VisionTransformer.__init__..)rdtypezQwen3-VL does not support z backend now.c s8g|]}tjjjtjd|dqS)z.blocks.)rrrrrrr)rr_rintermediate_sizer hidden_actrrrjrkrs  ))rdrer_rnum_position_embeddingsr\rspatial_merge_unitr]hasattrrrwnum_grid_per_siderlenrWr^ patch_embedr Embedding pos_embedrrrrotary_pos_embrmerger ModuleListrangedeepstack_merger_listrTrxget_default_dtype attn_backendr9 FLASH_ATTN TORCH_SDPA ROCM_AITER_FA RuntimeErrordepthblocks)rgrrrrhead_dimrhrrkre9sz         z Qwen3_VisionTransformer.__init__cC |jjjjSr)rrfweightrrgrjrjrkr zQwen3_VisionTransformer.dtypecCrr)rrfrdevicerrjrjrkrrzQwen3_VisionTransformer.devicei)maxsizehwrcCstt||d||f}||}||}|||||}|dddd}|}tt|d|||f}|||||}|dddd}|}ttj||gddS)Nr:rrYrZrnaxis) np broadcast_toarangereshape transposeflattenrx from_numpystack)rrrhpos_idsh_divw_divwpos_idsrjrjrk rot_pos_idss*z#Qwen3_VisionTransformer.rot_pos_idsgrid_thwcsrtdd|D}fdd|D}tj|ddjjdd}j|\}}||d }||d }||fS) Ncss |] \}}}t||VqdSrmax)rrrrrjrjrk sz6Qwen3_VisionTransformer.rot_pos_emb..csBg|]\}}}|dkr||jn ||j|dqS)r:)rrrepeat)rtrrrrjrkrs z7Qwen3_VisionTransformer.rot_pos_emb..rrT non_blockingr:)rrxcattorr get_cos_sinr)rgr max_grid_sizepos_idscossin cos_combined sin_combinedrjrrk rot_pos_embs z#Qwen3_VisionTransformer.rot_pos_embc#Cs|j}|j}|jj}g}|D]\}}}tjd|d|tj|jd} tjd|d|tj|jd} | tj } | tj } tj | d|dd} tj | d|dd}| | }| | }tj ||dd\}}tj | | dd\}}tj | |dd\}}||}||}||}d||}t ||||g}t ||||g}||}|| dd}tj ||||gdd  ddd}|j|jd }||} | |9} | jdd }!|! |||||||}!|!dd dd d dd|}!|!|dd d|}"||"qtj|dd S) Nrr:)rrrij)indexingrnrrrYrZ)rrr embedding_dimrxlinspacefloat32rrlongclampmeshgridrrrsumpermuteexpandappendr)#rgrrm_size hidden_dimoutputsrrrh_idxsw_idxsh_floorw_floorh_ceilw_ceildhdwdh_griddw_grid h_floor_grid w_floor_grid h_ceil_grid w_ceil_gridw11w10w01w00h_gridw_grid h_grid_idxindicesweightsembedscombinedrepeatedrjrjrkfast_pos_embed_interpolatesP        z2Qwen3_VisionTransformer.fast_pos_embed_interpolatercCsHtjg|jd}|jtjks|jtjkr"|dd|dd}|S)N)rr:rn)rxzerosrrr9rrr)rgrrrjrjrkcompute_attn_mask_seqlen s   z0Qwen3_VisionTransformer.compute_attn_mask_seqlenrlcCsp|j|j|jdd}||}t|tr|}tj|tjd}n| }| }| |}||}| |\}}t |dddf|dddf|dddfjdtjd}ttjdtjd|g}t|}|d}||} |j|jdd}g} t|jD]$\} } | ||||| d }| |jvr|j| } |j| |}| |q||}tj|g| dd }|S) NT)rrrrr:rYr)rrrrr)rrrr isinstancelistrarrayint32tolistnumpyr,rrcumsum concatenater-rxr unsqueezer. enumeraterrindexrrrr)rgrlr hidden_states grid_thw_list pos_embedsrrrrdeepstack_feature_lists layer_numblkdeepstack_merger_idxdeepstack_featurerjrjrkrssN   4        zQwen3_VisionTransformer.forwardr(c Csgd}t|jdd}t}|D]9\}}|D]\}}} ||vr!q|||}||} | j} | | || n||} t| dt} | | |||q|S)N)) attn.qkv.zattn.q.q)rBzattn.k.k)rBzattn.v.vF)remove_duplicate weight_loader)dictnamed_parameterssetreplacerGgetattrr!add) rgr(stacked_params_mapping params_dict loaded_paramsname loaded_weight param_name weight_nameshard_idparamrGrjrjrk load_weightsJs"      z$Qwen3_VisionTransformer.load_weights)rNr|)rtrurvrfloatrrrepropertyrxrr staticmethodrrwryrr0rr,r.rsrtuplerJrWrzrjrjrhrkr8sJb <   ,2rcsDeZdZddZdedefddZdedefddZdede fd d Z d d Z d ddde de de de dee BdBdeee ff ddZd-de de de ffdd Zde deee fde ffdd Zde deee fde fd d!Zd"ee ejBd#ed$e fd%d&Z  d.d'eeefd(ed)e dBd*edBdee f d+d,ZZS)/Qwen3VLProcessingInfocCs |jtSr)ctx get_hf_configrrrjrjrkr^gs z#Qwen3VLProcessingInfo.get_hf_configkwargsr`cKs |jjtfd|ddi|S)Nuse_fastT)r]get_hf_processorrpoprgr_rjrjrkrajs z&Qwen3VLProcessingInfo.get_hf_processorcK|jdi|jSNrj)raimage_processorrcrjrjrkget_image_processorqz)Qwen3VLProcessingInfo.get_image_processorcKrdre)ravideo_processorrcrjrjrkget_video_processortrhz)Qwen3VLProcessingInfo.get_video_processorcCst|jjd|dS)NT)video_needs_metadataexpected_hidden_size)rJr^rr_get_expected_hidden_sizerrjrjrkget_data_parserws  z%Qwen3VLProcessingInfo.get_data_parserrYT) num_frames do_resize image_width image_heightrorprfNcCs|dur |dkr |}n|dur|}t|t}|}|j}|j} |j} |j} |rX|r6t } || d} nt } i} | d||| | |j d|j dd| \}}t ||d}nt ||d}t || }t|| d}|j| }|j| }|||}|| d}||fS) Nr:)rotemporal_factor shortest_edge longest_edge)heightwidthfactor min_pixels max_pixels)rwrvrYrj)rjrgr/rr^rr\rr]video_smart_resizeimage_smart_resizesizer/r8rrvrw)rgrqrrrorprfis_video hf_configrr\ merge_sizer]r extra_kwargsresized_height resized_widthpreprocessed_sizepadded_num_framesgrid_tgrid_hgrid_w num_patchesnum_vision_tokensrjrjrk_get_vision_info~sH         z&Qwen3VLProcessingInfo._get_vision_info max_tokensstart_num_framescstj||dS)N)r)rd_get_max_video_frames)rgrrrhrjrkrsz+Qwen3VLProcessingInfo._get_max_video_framesseq_len mm_countscstj||tdS)N)max_frames_per_video)rd!get_num_frames_with_most_featuresDUMMY_VIDEO_NUM_FRAMES)rgrrrhrjrkrsz7Qwen3VLProcessingInfo.get_num_frames_with_most_featurescCs>|}|jd}|j||jd\}}|j||ddd}|S)NrurzrYrqrrrorf)rjr}!get_image_size_with_most_featuresr]get_num_video_tokens)rgrrrivideo_max_pixels target_width target_heightnum_video_soft_tokensrjrjrkget_max_video_tokenss  z*Qwen3VLProcessingInfo.get_max_video_tokensr' video_fpsrcsvt|ts |}t|dkr ||dgt|}fdd|DfddtdtDS)Nrrncsg|]}|qSrjrjridx)rrjrkrz?Qwen3VLProcessingInfo._calculate_timestamps..cs(g|]}||ddqS)r:rYrjri)r timestampsrjrkrs)r/r0r3rr)rgr'rrrj)rrrrk_calculate_timestampss  z+Qwen3VLProcessingInfo._calculate_timestampsmetadataout_itemdo_sample_frames sampled_fpsc Cs|}|j}|d}|d}|dur|dd}|rM|r|n|j}|d} t| |d|} ttt| |j|j| } t d| d|  t }||||} | S)Nframes_indicesfpsrFtotal_num_framesrr:)rjrgetrrwminr min_frames max_framesrrroundastyper3r) rgrrrrrirr'rrrorrjrjrk_get_video_second_idxs0  z+Qwen3VLProcessingInfo._get_video_second_idx)rYNN) rtrurvr^objectrrar rgrrjrnrwrr[r/rrrrrrr0rxryrXrrHr r+rrzrjrjrhrkr\fsp    7       r\c @szeZdZdeeefdefddZ ddedeeefdeeefdBdefdd Z d ed ed ed ede e f ddZ dS)Qwen3VLDummyInputsBuilderrr`cCs0|dd}|dd}d}d}||||S)Nimagervideo+<|vision_start|><|image_pad|><|vision_end|>+<|vision_start|><|video_pad|><|vision_end|>)r)rgr num_images num_videos image_token video_tokenrjrjrkget_dummy_texts  z(Qwen3VLDummyInputsBuilder.get_dummy_textNr mm_optionscCs|dd}|dd}|r|dnd}|r|dnd}|j\}} d} |rOt|ts0J|j} | rO| | kr@td| | | dkrJtd| t| | } t | d} |j } | j d} |jj| | j d\}}|jj ||| | d \}}|j|j}}|rt|tsJ|j}|r||krtd ||t||}|j}|r||krtd ||t||}|j|| ||d |j||| |d dS)NrrrrYz]video.num_frames override (%d) exceeds model's maximum number of frames (%d), will be ignoredzEvideo.num_frames override (%d) cannot be less than 2, will be ignoredrurrzMvideo.width override (%d) exceeds model's maximum width (%d), will be ignoredzOvideo.height override (%d) exceeds model's maximum height (%d), will be ignored)rwrvr overrides)rwrvror)rr)rinforr/rrologgerwarningrrrjr}r]rrwrv_get_dummy_images_get_dummy_videos)rgrrrrrimage_overridesvideo_overridestarget_image_widthtarget_image_heighttarget_num_framesnum_frames_overriderirtarget_video_widthtarget_video_heighttarget_video_sizerwidth_overrideheight_overriderjrjrkget_dummy_mm_datas          z+Qwen3VLDummyInputsBuilder.get_dummy_mm_datarwrvrorc Csjtj|||dfdtjd}g}t|D]}d|d|ddt|Dddd }||f} || q|S) NrZrg@cSsg|]}|qSrjrjrrjrjrkrz?Qwen3VLDummyInputsBuilder._get_dummy_videos..opencvF)rdurationrr video_backendr)rfulluint8rcopyr) rgrwrvrorr video_itemsrvideo_metadata video_itemrjrjrkrzs   z+Qwen3VLDummyInputsBuilder._get_dummy_videosr) rtrurvrrrwrrr(rr0r.rrjrjrjrkrs.   `rc seZdZdedeeefdeeefdeeefdef fdd Zded eeefdeeeffd d Z d e d eee fd e de efddZZS)Qwen3VLMultiModalProcessorpromptmm_data mm_kwargs tok_kwargsr`csHt|}|jjdi|}|dg}rg}g}|D]b} | \} tdi|} d| vr4dd| d<tdifddDt} | gg| d<gg| d<tjd| | |d} | d }|j |d }| d|d }| | d | | d qtt |t |d} nt} tj||||d}t|fi| }t|S)NvideosrFcsi|] }|dkr||qS)rrj)rrDrrjrk szAQwen3VLMultiModalProcessor._call_hf_processor..rr)rrrr input_idsrr:video_grid_thwpixel_values_videos)rrrj)rHrrarbrrrd_call_hf_processor tokenizer batch_decoderKrrxrr )rgrrrr processorrvideo_grid_thw_lstpixel_values_videos_lstitem video_arrayvideo_mm_kwargs video_mm_data video_outputsrvideo_placeholderprocessed_outputscombined_outputsrhrrkrsd    z-Qwen3VLMultiModalProcessor._call_hf_processor hf_inputshf_processor_mm_kwargscCst|jjj|Sr)rLrr^rr)rgrrrjrjrk_get_mm_fields_configs  z0Qwen3VLMultiModalProcessor._get_mm_fields_configmm_items out_mm_kwargsc sjjd ijjd i}jj}|j|j |j|jddt ffdd }dt f f dd }t dj |dt d d |dgS) NrYitem_idxcsBd|}|dj}t|tjsJt|}jg|S)Nrimage_grid_thw)datar/rxryrwprodimage_token_id)rrr num_tokens) hf_processor merge_lengthrrjrkget_image_replacement_qwen3vls   zUQwen3VLMultiModalProcessor._get_prompt_updates..get_image_replacement_qwen3vlcsd|}|dj}t|tjsJd|\}}d}d}t|tr.||}j||||}t ||dksNJdt |d|ddfd d |D}t |d d fd d |D} jj j} | dur| d krtt || } t |dkrg} nAt |d krg} n7} t| | d} | t |d }| t |d }| g} td t |D]}||d |krd nd}| |qg}t|D]#\}}||| |t | kr|nd}| gg|gqt|S)NrrrrrzThe timestamps length(z ) should be equal video length (z).cs$g|]}jd|ddddqS)F)add_special_tokens)encode)r curr_time)rrjrkrsziQwen3VLMultiModalProcessor._get_prompt_updates..get_video_replacement_qwen3vl..r:csg|]}qSrjrjrrtokens_per_framerjrkrrrn)rr/rxryrr7rXrrrrwrr] get_mm_configvideo_pruning_rater%rrrr8extendr5select_token_id)rrrrrrrrframes_idx_tokenper_frame_token_countsr total_retainedfirst_frame_tokensremaining_tokensbase remainder frame_idxextra placeholdertimestamp_tokenstokens_this_frame) rrrrrgrvideo_token_idvision_end_token_idvision_start_token_idr rkget_video_replacement_qwen3vlsl            zUQwen3VLMultiModalProcessor._get_prompt_updates..get_video_replacement_qwen3vlr)modalitytarget replacementrrrj) rrarg get_tokenizerr^rrrrrwr3r)rgrrrrfrrrrj) rrrrrrgrrrrrk_get_prompt_updatess*   "=z.Qwen3VLMultiModalProcessor._get_prompt_updates)rtrurvrrrr rr*rr0r r,rr4r$rzrjrjrhrkrs8   M     rrnr positionsintermediate_tensors inputs_embedsdeepstack_input_embeds)dynamic_arg_dimscsreZdZdddedeffddZ   ddejdBd ejd edBd ejdBd edBd ejeBf ddZ Z S) Qwen3LLMModelr|r vllm_configrcsPtj||d|jjj}tjs"t|dr$|jt |j ks&JddSdSdS)Nr-rrzLstart_layer should be greater than or equal to len(deepstack_visual_indexes)) rdre model_configrrr is_first_rankr start_layerrr)rgr-rrrhrjrkre[s  zQwen3LLMModel.__init__Nrr&r'r(r)r`c Cstjr|dur |}n||}d}n|dusJ|d}|d}g}tt|j|j|jD].\} } | |jvr>| ||| |||\}}|dur\| t dt |vr\||d| }q.tj sht ||dS|||\}} t |dkrz||fS|S)Nr:residualrdeepstack_input_embeds_)r:r2)rr0embed_input_idsr r8layersr1 end_layeraux_hidden_state_layersrrr is_last_rankr6r) rgrr&r'r(r)r:r2aux_hidden_statesrlayerrrjrjrkrsfsF       zQwen3LLMModel.forward)NNN) rtrurvrrrerxryr6rsrzrjrjrhrkr+Ns$ r+cs,eZdZdddedeffddZZS)Qwen3LLMForCausalLMr|r,r-rcstt||jjj}|j}||_||_t|t |dd|_ t j r8|j r,|j j|_nt|j|j|dd|_nt|_t|j|_|j j|_dS)Nmodelr.lm_head)rr)rdrMrer/r text_configrconfigr+rSr<rr8tie_word_embeddings embed_tokensr=r vocab_sizer_rPrlogits_processormake_empty_intermediate_tensors)rgr-rr?rrhrjrkres*     zQwen3LLMForCausalLM.__init__)rtrurvrrrerzrjrjrhrkr;s$r;)r dummy_inputscseZdZgdddgdgdZdZeddd d d Zed ed e dedBfddZ ddde deffddZ de e dfddfddZde e dffddZde dedBfdd Zd!ejddfd"d#Zde ddfd$d%Zd&ededBfd'd(Zd&ededBfd)d*Zd+ede ejdffd,d-Zd.ede ejdffd/d0Zd1e ejdfd+ede ejdffd2d3Zd4e ejdfd.ede ejdffd5d6Zd&edefd7d8Z d9e!e d:e!e"de#e e e e ffd;d<Z$d=e%d>e de!ejdBfd?d@Z&d=e%d>e de!e dBfdAdBZ'd=e%d>e de!e dBfdCdDZ(dEe!e dFe ejdfdGej)dHe de e ejdfeje ff dIdJZ*d9e!e d:e!e"de eje ffdKdLZ+d&ede,dBfdMdNZ-dOejdFe,dPejde eje,ffdQdRZ. djddSdTdEejdFe,dBdPejdBdUe/dejf dVdWZ0  dkdEejdBdXejdYedBdOejdBd&edejeBf dZd[Z1d\ejdejdBfd]d^Z2d_e3e eejfde4efd`daZ5de6fdbdcZ7dde de fdedfZ8dge de fdhdiZ9Z:S)lQwen3VLForConditionalGeneration)q_projk_projv_proj gate_projup_projqkv)qkv_proj gate_up_projrLTvisual.zlanguage_model.lm_head.zlanguage_model.model.)z model.visual.zlm_head.zmodel.language_model.)orig_to_new_prefixr rr`NcCs$|drdS|drdStd)Nrrrrz)Only image or video modality is supported) startswith ValueError)clsr rrjrjrkget_placeholder_strs  z3Qwen3VLForConditionalGeneration.get_placeholder_strr<r,r-rcsHtjjj}jj}|_||_|jdk|_|j |_ | |_ t j d|_ |j r5tj jnd|_j j|_|j|j|_|ddh*tj tdd|t|dd |_|j rofd d t|jD|_Wdn1sywY|tt|d d |_Wdn1swY|jj|_dS)Nrrrrr rms_norm_epsrvisual)rrrcs g|] }tjjjjqSrj)rxr-scheduler_configmax_num_batched_tokensr>r_rr?r-rjrkr sz.language_modelr.)rdrer/rrmultimodal_configr?mm_encoder_tp_moderr is_multimodal_pruning_enabledrr use_deepstackrrdeepstack_num_levelr visual_dimmultiscale_dim_mark_tower_modelrrLrSrVrr)_mark_language_modelr;rZrD)rgr-rrr[rhrYrkresJ        z(Qwen3VLForConditionalGeneration.__init__r5.cCs||jj_dSr)rZr<r7)rgr5rjrjrkset_aux_hidden_state_layerssz;Qwen3VLForConditionalGeneration.set_aux_hidden_state_layerscCs t|jjj}d|d|dfS)NrYrZ)rrZr<r5)rg num_layersrjrjrk"get_eagle3_aux_hidden_state_layersszBQwen3VLForConditionalGeneration.get_eagle3_aux_hidden_state_layersrcs.tddsdStfddtjDS)Nr)cs&i|]}d|j|dqS)r3N)r)rrrgrjrkr,s zOQwen3VLForConditionalGeneration._get_deepstack_input_embeds..)rLr6rr_)rgrrjrgrk_get_deepstack_input_embeds#s  z;Qwen3VLForConditionalGeneration._get_deepstack_input_embedsr)csztddsdS|djddkr%fddtjD_tjD]}j|d||q*dS)Nr)r:rcs4g|]}tjjjjjdjjdjdqS)rrr)rxr-r?r>r_r)rrrrgrjrkr;s  zOQwen3VLForConditionalGeneration._set_deepstack_input_embeds..)rLr}r)rr_copy_)rgr)rrjrgrk_set_deepstack_input_embeds4s    z;Qwen3VLForConditionalGeneration._set_deepstack_input_embedscCsFt|ddsdS|dkrt|jD]}|j|d|qdSdS)Nr)r)rLrr_r)zero_)rgrrrjrjrk_clear_deepstack_input_embedsIs z=Qwen3VLForConditionalGeneration._clear_deepstack_input_embedsr_cKsh|dd}|dd}|dd}|dur|durdS|dur'td||dS|dur2td||dSdS)N pixel_values image_embedsr)typernr)rpror)rbrFrD)rgr_rnrorrjrjrk_parse_and_validate_image_inputRs$   z?Qwen3VLForConditionalGeneration._parse_and_validate_image_inputcKsv|dd}|dd}|dd}|dd}|dur"|dur"dS|dur.td|||dS|dur9td||dSdS)Nr video_embedsrsecond_per_grid_ts)rprrrs)rprrr)rbrIrG)rgr_rrrrrsrjrjrk_parse_and_validate_video_inputjs(    z?Qwen3VLForConditionalGeneration._parse_and_validate_video_input image_inputcCs|d}|jdks J|ddkr|d|jj}n|d|jj}|jr2t|j||ddS|j||d}|jj}|d ||}| |S) NrrYrprornrope_3d rope_typerrn) ndimrprVrrrVr3rrsplit)rgrurrornrsizesrjrjrk_process_image_inputs  z4Qwen3VLForConditionalGeneration._process_image_input video_inputcCs|d}|jdks J|ddkr|d|jj}n |d|jj}|jr4|}t|j||ddS|j||d}|jj}|d ||}| |S) NrrYrprrrrvrwryrn) rzrprVrrr3rVrrr{)rgr~rrrrr;rr|rjrjrk_process_video_inputs    z4Qwen3VLForConditionalGeneration._process_video_inputimage_embeds_splitc Csj|jj}|d}|}g}t||D]\}}t|||j} tj|| gdd}| |q|}t |S)a Append mrope positions for each for images. This is necessary to recover correct mrope positions after video pruning Args: image_embeds_split: Tuple of image embeddings for each image item. image_input: Image input data. Returns: Tuple of image embeddings for each image item. Resulting embeddings will have extra 4 channels for computed mrope positions. rr:r) rVrr3zipr$rrrxrrr[) rgrrurrr;image_embeds_outembr}r&rjrjrk_postprocess_image_embeds_evss z=Qwen3VLForConditionalGeneration._postprocess_image_embeds_evsvideo_embeds_splitc Cs0|d}|jdks J|}|jj}|d}|dur'tjt|tjd}n|}t |j j dd}g}t |||D]X\} } } t | | |jj|jd} td | jd | | d | d | d|jd | d t| ||| d | j} | | } | | } tj| | gd d} || q;t|S)a Prunes video embeddings via Efficient Video Sampling (EVS) and then appends mrope positions for each retained embeddings Args: video_embeds_split: Tuple of video embeddings for each video item. video_input: Video input data. Returns: Tuple of video embeddings for each video item. Resulting embeddings will have extra 4 channels for computed mrope positions. rrYrsNrtokens_per_secondg?)rrCz\EVS: Video tokens pruned from %d to %d (T=%d,H=%d,W=%d, pruning_rate=%.2f, reduction=%.1f%%)rr:d)rvideo_second_per_gridr)rzr3rVrrrxonesrrrLr?rrr&r rdebugror rrXmeanr$rrrrr[)rgrr~rr;rrsrvideo_embeds_outrr}video_second_per_grid_tretention_maskr&rjrjrk_postprocess_video_embeds_evssT    z=Qwen3VLForConditionalGeneration._postprocess_video_embeds_evscKsZi}|D]&}|dvrd|vr|jdi||d<|dvr*d|vr*|jdi||d<q|S)N)rnror)rrrrrj)rqrt)rgr_mm_input_by_modality input_keyrjrjrk%_parse_and_validate_multimodal_inputss  zEQwen3VLForConditionalGeneration._parse_and_validate_multimodal_inputs input_tokens mm_featuresccsZ|jj}|jjj}t|dddD]}|jj}|jdkr>|jdj \}}} |dks3Jd||||| |fVq|jdkr|jd j \}}} ||} | |} t |d od|j d uod|j d k} | r| |j|} | d ur| D] }||| | fVqtqt d |j dt|D]}|||}|| | fV|| | 7}qqtd|jd S)a Iterate over multimodal features and yield grid information. For videos with EVS (Efficient Video Sampling) enabled, this function computes the offset based on the pruned token count rather than relying on input_tokens.index(), which would fail when tokens are pruned. Args: input_tokens: List of token IDs in the prompt mm_features: List of multimodal feature specifications Yields: Tuple of (offset, grid_h, grid_w) for each frame/image cSs|jjSr) mm_positionoffset)frjrjrk@szAQwen3VLForConditionalGeneration.iter_mm_grid_hw..)keyrrr:zImage must have 1 frame, got rrr Nr zEVS is enabled (pruning_rate=z[) but is_embed mask is missing from mm_position. This indicates a bug in prompt processing.zUnsupported modality: )r?rrrsortedrrr rr3rr  _extract_frame_offsets_from_maskrrr9rR)rgrrrr mm_featurerrrr llm_grid_h llm_grid_wis_evs_enabled frame_offsets rel_offsetrrjrjrkiter_mm_grid_hw-sH        z/Qwen3VLForConditionalGeneration.iter_mm_grid_hwrexpected_framesc Cst|dd}|dur dStj|tjdd}tj|dd}|dkr(dS|dkr2|g}n%t|}tj|dkdd}|dkrL|g}n t || d }t ||krht d t ||dS|d|S) aExtract contiguous segments from EVS is_embed mask. The EVS (Efficient Video Sampling) mask marks which placeholder positions should be filled with video embeddings. This method splits the mask into contiguous segments, where each segment represents one retained frame. This is a pure function - it does not modify any state and always returns the same output for the same input (idempotent). Args: mm_position: MultiModal position containing the is_embed mask expected_frames: Expected number of frame segments Returns: List of tensors, each containing indices for one frame segment, or None if EVS is not enabled or validation fails. is_embedNrrnF)as_tuplerr:z8EVS mask segments (%d) do not match expected frames (%d))rLrx as_tensorrrpnonzerornumeldiff tensor_splitrMr3rrr) rgrr is_embed_mask mask_tensor true_indicessegmentsdiffs split_pointsrjrjrk_get_evs_mask_segmentsls0       z6Qwen3VLForConditionalGeneration._get_evs_mask_segmentscC&|||}|dur dSdd|DS)aReturn relative offsets for each EVS-retained frame. The prompt processor stores a boolean mask inside ``mm_position`` that marks which placeholder locations should be populated with video embeddings. By splitting that mask into contiguous runs we can recover the start of every retained frame without probing ``input_tokens``. Args: mm_position: MultiModal position containing the is_embed mask expected_frames: Expected number of frames Returns: List of starting offsets (relative to mm_position) for each frame, or None if EVS is not enabled. NcSsg|] }t|dqS)r)rwr)rsegmentrjrjrkrszTQwen3VLForConditionalGeneration._extract_frame_offsets_from_mask..rrgrrrrjrjrkrs z@Qwen3VLForConditionalGeneration._extract_frame_offsets_from_maskcCr)aReturn actual token count for each EVS-retained frame. This function calculates the actual number of tokens per frame by analyzing the is_embed mask, accounting for EVS pruning. Each frame may have a different token count due to content-aware pruning. Args: mm_position: MultiModal position containing the is_embed mask expected_frames: Expected number of frames Returns: List of token counts for each frame, or None if EVS is not enabled. NcSg|]}t|qSrjr)rsegrjrjrkrrzRQwen3VLForConditionalGeneration._get_actual_frame_token_counts..rrrjrjrk_get_actual_frame_token_countss z>Qwen3VLForConditionalGeneration._get_actual_frame_token_countsrmultimodal_embeddingsmrope_positionsnum_computed_tokenscCs|jj}|jj}|jj}t|r|djn|j}tj||tjd} dd|D} dd|D} t | | |||||\} } t | | | fS)a& Update part of input mrope positions (starting with num_computed_tokens index). Original mrope_positions are computed for unpruned sequence and becomes incorrect once pruning occurs, so once we prune media tokens we should reflect this in the mrope_positions before we feed it to LLM. Args: input_ids: (N,) All input tokens of the prompt (Containing entire sequence). multimodal_embeddings: Tuple of multimodal embeddings. mrope_positions: Existing mrope positions (3, N) for entire sequence num_computed_tokens: A number of computed tokens so far. Returns: Tuple of (multimodal_embeddings, mrope_positions, mrope_position_delta). rricSs g|] }|ddddfqS)Nrjrmmrjrjrkrs zMQwen3VLForConditionalGeneration.recompute_mrope_positions..cSs,g|]}|ddddfddqS)Nrr:r)r rrrjrjrkrs ) r?rrrrrrxrrr'r[)rgrrrrrrrr input_ids_tmm_embeddings_outmm_embeddings_posr&mrope_positions_deltarjrjrkr's,  z9Qwen3VLForConditionalGeneration.recompute_mrope_positionscCsni}|D]7}|jdkr;t|do|jduo|jdk}|r;|jdjd}||j|}|dus5Jd|||jj<qg}d} i} |||D]\} } } | | }t |dkr_|d dnd}d}|D]}| |krm|}qe|dur||vs~Jd |d || vrd| |<||}| |}|t |ksJd |d t |d ||}| |d7<n| | }t t |d|f|}||||7}t d| | fdd}|ddd|f|}||| |} qH| t |krt |dkr|d dnd}t || }t t |d|f|}||t j|dddd}| dt |}t||fS)Nrr r rrzGEVS enabled but failed to extract frame token counts from is_embed maskrnr:zFound base_offset z" but not in frame_token_counts_mapzEVS frame index z out of range (total frames: )rZr)r rr rr3rrrrrrrrrrr'rr6rrxr)rgrrframe_token_counts_maprrr token_countsllm_pos_ids_liststframe_counts_idxrrrtext_lenst_idx base_offset feat_offsetcountsractual_frame_tokenstext_positions grid_indicesframe_positionsfinal_text_positions llm_positionsmrope_position_deltarjrjrkget_mrope_input_positions s|             z9Qwen3VLForConditionalGeneration.get_mrope_input_positionscKs|jdi|}|s dSd}|D]6}||}|dkr.||}|jr(|||}|t|7}|dkrF||}|jr@|||}|t|7}q|S)Nrjrr)rr}r]rr[rr)rgr_rrr multimodal_inputimage_embeddingsvideo_embeddingsrjrjrkembed_multimodalgs,    z0Qwen3VLForConditionalGeneration.embed_multimodalr( is_multimodalc Csdd|D}tj|dd}tj||j|jgdd\}}tj||dd}tj||dd}||d|j|d}t|||d}| |j d|j|j}| ddd}||fS) NcSrrjr)rrlrjrjrkrrzMQwen3VLForConditionalGeneration._compute_deepstack_embeds..rrrnr:r(rrrY) rxrr{r`ra new_zerosr}r_rRrpror ) rgr(rr visual_lensmultimodal_embeddings_catmultimodal_embeddings_main multimodal_embeddings_multiscaler)rjrjrk_compute_deepstack_embedss: z9Qwen3VLForConditionalGeneration._compute_deepstack_embedsFrhandle_oov_mm_tokenrcCsz|j||jj||d}|dust|dkr|St|}|jr)|j|||d\}}nd}t|||d}|dur;|||S)Nrrr) _embed_text_input_idsrZr4rrBr^rrRrk)rgrrrrr(r)rjrjrkr4s6 z/Qwen3VLForConditionalGeneration.embed_input_idsr&r'cKsl|durd}|durtjr||d}nd}|jj|||||d}|dur4tjr4||d|S)aRun forward pass for Qwen3VL. Args: input_ids: Flattened (concatenated) input_ids corresponding to a batch. positions: Flattened (concatenated) position ids corresponding to a batch. **NOTE**: If mrope is enabled (default setting for Qwen3VL opensource models), the shape will be `(3, seq_len)`, otherwise it will be `(seq_len,). intermediate_tensors: Intermediate tensors from previous pipeline stages. inputs_embeds: Pre-computed input embeddings. **kwargs: Additional keyword arguments including: - pixel_values: Pixel values to be fed to a model. `None` if no images are passed. - image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM. `None` if no images are passed. - pixel_values_videos: Pixel values of videos to be fed to a model. `None` if no videos are passed. - video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM. `None` if no videos are passed. Nrr%)rr0rhr}rZr<rm)rgrr&r'r(r_r)r:rjrjrkrss"  z'Qwen3VLForConditionalGeneration.forwardr:cCs |j|Sr)rZcompute_logits)rgr:rjrjrkrs z.Qwen3VLForConditionalGeneration.compute_logitsr(cCst|}|j||jdS)N)mapper)rOrWhf_to_vllm_mapper)rgr(loaderrjrjrkrWsz,Qwen3VLForConditionalGeneration.load_weightscCstjdddgddS)z< Get the module prefix in multimodal models rZz visual.mergerzvisual.deepstack_merger_listrO)rZ connector tower_model)r"from_string_fieldrrjrjrkget_mm_mappings z.Qwen3VLForConditionalGeneration.get_mm_mappingnum_image_tokenscCs|j}|j}|j}||dSNrYr?rr)rgrrrrrjrjrkget_num_mm_encoder_tokens$s z9Qwen3VLForConditionalGeneration.get_num_mm_encoder_tokensrcCs|j}|j}|j}||dSrr)rgrrrrrjrjrkget_num_mm_connector_tokens.s z;Qwen3VLForConditionalGeneration.get_num_mm_connector_tokensrr);rtrurvpacked_modules_mappingsupports_encoder_tp_datarQr classmethodrrwrTrrer[rdrfr6rhrxryrkrmrrErqrHrtr}rrrrHrr0r)rrr-rrr LongTensorr'rr;rrrr4rsrrrJrWr"rrrrzrjrjrhrkrFsH 2              H ?  7      9  ]  , / 8 $   rF)__doc__collections.abcrrrrr functoolsrr itertoolsr typingr r4rrxtorch.nnrtorch.nn.functional functionalr transformersr transformers.models.qwen2_vlr 6transformers.models.qwen2_vl.image_processing_qwen2_vlr r|transformers.models.qwen3_vlrr3transformers.models.qwen3_vl.configuration_qwen3_vlrr6transformers.models.qwen3_vl.video_processing_qwen3_vlr{transformers.video_utilsrvllm.compilation.decoratorsr vllm.configrvllm.config.multimodalrrvllm.distributedr vllm.loggerr%vllm.model_executor.layers.activationrvllm.model_executor.layers.convr!vllm.model_executor.layers.linearrr+vllm.model_executor.layers.logits_processorr'vllm.model_executor.layers.quantizationr+vllm.model_executor.layers.rotary_embeddingr3vllm.model_executor.layers.vocab_parallel_embeddingr -vllm.model_executor.model_loader.weight_utilsr!)vllm.model_executor.models.module_mappingr"vllm.multimodalr#vllm.multimodal.evsr$r%r&r'vllm.multimodal.inputsr(r)r*r+r,r-r.vllm.multimodal.parser/r0vllm.multimodal.processingr1r2r3r4r5 vllm.sequencer6vllm.utils.collection_utilsr7vllm.utils.math_utilsr8#vllm.v1.attention.backends.registryr9 interfacesr;r<r=r>r?r@rArB qwen2_5_vlrCrDrErFrGrHrIqwen2_vlrJrKrLqwen3rMrNutilsrOrPrQrRrSvisionrTrUrVrtrrrrWr{rrrr\rrr+r;register_processorrFrjrjrjrks                     $     ($  %440+= @!