o iz @s|UddlZddlmZmZmZddlmZddlmZm Z ddl m Z m Z m Z mZddlZddlmZddlmZddlmZddlmZdd lmZdd lmZmZdd lmZdd lm Z dd l!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.m/Z/ddl0m1Z1m2Z2ee3Z4ee5e5dBfZ6 eGdddZ7GdddZ8dddddede5dedBde9e5dBdej:f d d!Z;ed"ejde9ej<ejde9ej<ejZDGd?d@d@ej:ZEeddAdBej:dCeFej:eGeFej:dDfBdBfdEdFZHeddAdBej:dGeej:gej:fdCeFej:eGeFej:dDfBdBfdHdIZIGdJdKdKe ZJGdLdMdMejjKZLdaMdaNdNeAddfdOdPZOdBejj:dejj:fdQdRZPdSeAdTeJde5deGeAeAejjQffdUdVZRiZSeTeAe9e5feUdW<dXejj:de9e5fdYdZZVd[e5dXejj:de>fd\d]ZWd^e9e5d_eAfd`daZXde5d[e5de5fdbdcZYdede dBfdddeZZddge5dheAdeAfdidjZ[ kddlej|S)N) r!itemsreplacer" startswithr#endswithjoinrsplit)r&r*substrnew_keyprefixsuffixr'r'r( _map_name@s&  zWeightsMapper._map_nameweightscfdd|DS)Nc3s.|]\}}|dur|fVqdSNr6).0namedataout_namer&r'r( [sz&WeightsMapper.apply..r')r&r7r'r>r(applyXs zWeightsMapper.applyvaluescr8)Ncs"g|] }|durqSr9r:)r;r<r>r'r( bs z,WeightsMapper.apply_list..r'r&rBr'r>r( apply_listas zWeightsMapper.apply_listcsfdd|DS)Ncs(i|]\}}|dur|qSr9r:)r;r<valuer>r'r( is z,WeightsMapper.apply_dict..)r,rDr'r>r( apply_dicths zWeightsMapper.apply_dict)r$rr%r)__name__ __module__ __qualname____doc__rdictr!WeightsMapping__annotations__r"r#r)strr6rtupletorchTensorrAlistrErrHr'r'r'r(r0s    &rc seZdZdZgdZddddddejdeedBdeedBdeedBd eedBd df fd d Z d e e ee j fd e e ee e ee j fffddZdeded efddZded efddZded efddZdedejd e e ee j fd e efddZdejdeee j ffddZdedejd e e ee j fd e efd d!Zedd"d e e ee j fd#edBd eefd$d%ZZS)&AutoWeightsLoadera" Helper class to load weights into a [`torch.nn.Module`][]. It is able to automatically detect child modules and parameters while iterating over the weights only once. The weight loading logic for individual modules can be overridden by defining a `load_weights` method. Similarly, the weight loading logic for individual parameters can be overridden by defining a `weight_loader` method. Detailed weight loading information can be viewed by setting the environment variable `VLLM_LOGGING_LEVEL=DEBUG`. )zrotary_pos_emb.inv_freqzrotary_emb.inv_freqzrotary_emb.cos_cachedzrotary_emb.sin_cachedN) skip_prefixes skip_substrsignore_unexpected_prefixesignore_unexpected_suffixesmodulerVrWrXrYr%csLt||_|p g|_|pg|_|pg|_|pg|_|j|j7_dSr9)super__init__rZrVrWrXrYROTARY_EMBEDS_UNUSED_WEIGHTS)r&rZrVrWrXrY __class__r'r(r\s     zAutoWeightsLoader.__init__r7ccsDdd|D}tj|dddD]\}}|dd|DfVqdS)Ncss$|] \}}|dd|fVqdS).r+N)split)r; weight_name weight_datar'r'r(r@s  z4AutoWeightsLoader._groupby_prefix..cSs |ddSNrr'xr'r'r( z3AutoWeightsLoader._groupby_prefix..)r*css0|]\}}t|dkrdn|d|fVqdS)r+N)len)r;parts weights_datar'r'r(r@s  ) itertoolsgroupby)r&r7weights_by_partsr4groupr'r'r(_groupby_prefixsz!AutoWeightsLoader._groupby_prefixr4restcCs&|dkr|S|dkr |Sd||fS)Nrir`)r0)r&r4rrr'r'r( _get_qualnames zAutoWeightsLoader._get_qualnamequalnamecs0tfdd|jDptfdd|jDS)Nc3|]}|VqdSr9r.r;prtr'r(r@z.AutoWeightsLoader._can_skip..c3s|]}|vVqdSr9r')r;r2ryr'r(r@s )anyrVrW)r&rtr'ryr( _can_skips$zAutoWeightsLoader._can_skipcs8fdd|jD}fdd|jD}t|pt|S)Nc3rur9rvrwryr'r(r@rzz;AutoWeightsLoader._can_ignore_unexpected..c3rur9)r/)r;sryr'r(r@rz)rXrYr{)r&rtiupiusr'ryr(_can_ignore_unexpectedsz(AutoWeightsLoader._can_ignore_unexpected base_prefixparamccs|D]F\}}|||}||rtd|q|dkr3||r)td|qtd|d|t|dt}|||td||j|VqdS)NzSkipping weight %srizIgnoring weight %sz Attempted to load nested weight z into a single parameter weight_loaderzLoaded weight %s with shape %s) rsr|loggerdebugr ValueErrorgetattrrshape)r&rrr7rbrcweight_qualnamerr'r'r( _load_params(        zAutoWeightsLoader._load_param child_paramsc CsLt|tjtjtjtjtjtjtjfr"| }dD] }||||<qdSdS)z Add tensor names that are not in the model params that may be in the safetensors, e.g., batch normalization stats. ) running_mean running_varnum_batches_trackedN) isinstancenn BatchNorm1d BatchNorm2d BatchNorm3dLazyBatchNorm1dLazyBatchNorm2dLazyBatchNorm3d SyncBatchNorm state_dict)r&rZrmodule_state_dict stat_namer'r'r(_add_loadable_non_param_tensorss   z1AutoWeightsLoader._add_loadable_non_param_tensorsc #st|ttfr dS|jkr5t|dd}t|r5||}|dur(td|n tfdd|EdHt | }t |j dd} || |D]\}} |} ||vrw| drjtd| qN| ||| EdHqN||vr| rtd | qN| ||| EdHqN| d} | } | s| rtd | qN| d} | }| s|rtd | qNfd d |j ddD}d| djdd|d| }t|dS)N load_weightsz1Unable to collect loaded parameters for module %scs |Sr9)rsrerr&r'r(rgrhz0AutoWeightsLoader._load_module..F)recurser`zSkipping module %szSkipping param %szSkipping missing %szIgnoring missing %scsh|]\}}|qSr'r')r;k_)rr'r( =s z1AutoWeightsLoader._load_module..Tz&There is no module or parameter named z in z(. The available parameters belonging to z (z) are: )rStageMissingLayerPPMissingLayerrZrcallablerwarningmaprMnamed_childrennamed_parametersrrqrsr|r _load_modulerr _get_namer)r&rrZr7module_load_weights loaded_params child_modulesr child_prefix child_weightsr4can_skip_modulecan_skip_paramcan_ignore_modulecan_ignore_paramdesc_param_keysmsgr'rr(rsv                   zAutoWeightsLoader._load_module)mapperrcs<|dur ||}fdd|D}tdj|}|S)Nc3s&|]\}}|s||fVqdSr9)r|)r;r<weightr&r'r(r@Rs  z1AutoWeightsLoader.load_weights..ri)rAsetrrZ)r&r7rautoloaded_weightsr'rr(rHs  zAutoWeightsLoader.load_weights)rIrJrKrLr]rModulerTrPr\rrQrRrSrqrsboolr|r ParameterrrMrrrrrr __classcell__r'r'r^r(rUpsx           OrUri)r4 hf_config architectures vllm_configr4rrr%cCsFddlm}|dur|dur|jj}|dur|j||d}|||dS)z Helper function to initialize an inner model registered to vLLM, based on the arguments passed to the outer vLLM model. r)initialize_modelN)r)rr4)&vllm.model_executor.model_loader.utilsr model_configrwith_hf_config)rr4rrrr'r'r(init_vllm_registered_modelZs  rrfcCdSr9r'rer'r'r( flatten_bnqrcCrr9r'rer'r'r(rurconcatTcCrr9r'rfrr'r'r(ryF)rcCrr9r'rr'r'r(rrcCs4t|tjr |ddS|rt|Sdd|DS)z Flatten the `B` and `N` dimensions of batched multimodal inputs. The input tensor should have shape `(B, N, ...)`. rr+cSsg|] }|D]}|qqSr'r')r;x_bx_nr'r'r(rCszflatten_bn..)rrRrSflattencatrr'r'r(rs   embeddingscCs0t|tjr |ddSttdd|DS)z` Recursively flattens and concatenates NestedTensors on all but the last dimension. rcs|]}t|VqdSr9)_flatten_embeddings)r;tr'r'r(r@z&_flatten_embeddings..)rrRrSrrrQrr'r'r(rs  rcCs>t|tjrddd|jddDSddd|DS) ze Constructs a debugging representation of the number of embeddings in the NestedTensors. z x cSsg|]}t|qSr')rP)r;dimr'r'r(rCsz/_embedding_count_expression..Nz + csrr9)_embedding_count_expression)r;innerr'r'r(r@rz._embedding_count_expression..)rrRrSr0rrr'r'r(rs rlstintervalcCsBddtt||dD}|D] }||}|||q|S)NcSsg|]}gqSr'r'r;rr'r'r(rCsz*split_list_into_ranges..r+)rangemaxappend)rrrangesnumindexr'r'r(split_list_into_rangess r inputs_embedsmultimodal_embeddings is_multimodalc Cst|dkr|St|}|j}z||d|j|dW|StyQ}z&t|}|}||krHt |}t d|d|d|d|t d|d }~ww) z Merge `multimodal_embeddings` into `inputs_embeds` by overwriting the positions in `inputs_embeds` corresponding to placeholder tokens in `input_ids`. Note: This updates `inputs_embeds` in place. rr)dtypezAttempted to assign z = z multimodal tokens to z placeholdersz%Error during masked scatter operationN) rjrrmasked_scatter_ unsqueezeto RuntimeErrorsumitemrr) rrrmm_embeds_flat input_dtypeenum_actual_tokensnum_expected_tokensexprr'r'r(_merge_multimodal_embeddingss0   relementstest_elements_listcCs(tj|tdj|jdd}t||S)N) pin_memoryT)device non_blocking)rRtensorrrrisin)rr test_elementsr'r'r( isin_lists rcsVeZdZddedejdBddffdd Zdefdd Zd d Zdefd d Z Z S)rN stage_namerZr%cst||_||jd<dSNrZ)r[r\r__dict__)r&rrZr^r'r(r\s zStageMissingLayer.__init__r<cCst|jd|Sr)rr)r&r<r'r'r( __getattr__szStageMissingLayer.__getattr__cOst|d)Nz should not be called)rr&argskwargsr'r'r(__call__zStageMissingLayer.__call__cCs d|jS)Nz stage_name=)rrr'r'r( extra_reprs zStageMissingLayer.extra_reprr9) rIrJrKrPrrr\rrr rr'r'r^r(rs " r)targetsrZr .c#stt|dur4dtjdtdtjffdd }t| VWddS1s-wYdSVD]\}}t||rI|q;dS)aA Within this context, collect all direct child assignments to `module`, returning a list of children names that is internally updated until the context is exited. If `targets` is set, instead collect descendents of `module` that are an instance of `targets`, even if they aren't direct children. Nmodule_r< submodulecs|ur |dSdSr9)rr r<r children_namesrZr'r(hookszcollect_children..hook)rTrPrrr named_modulesrr)rZr rr<r r'rr(collect_children s   "  r placeholderc #sdurLdtjdtdtjffdd }t|)td dVWdn1s-wYWddSWddS1sEwYdSdtjdtdtjffdd }t| dVWddS1sowYdS) al Within this context, prevent weight initialization from using device memory and replace direct child assignments to `module` with the result of `placeholder()`. If `targets` is set, instead prevent weight initialization and replace assignments where the child is an instance of `targets`, even if they aren't direct children of `module`. Nr r<r cs|ur|S|Sr9r'r )rZrr'r(r;szno_init_weights..hookmetacs4t|r |dt|r|d|S|S)Nr)rrr )rr r'r(rEs    )rrrPr rRr)rZrr rr')rZrr r(no_init_weights*s P  "rc@s"eZdZdedejjfddZdS)LayerFnr4r%cCrr9r')r&r4r'r'r(rUszLayerFn.__call__N)rIrJrKrPrRrrrr'r'r'r(rTsrcs(eZdZdZfddZddZZS)rzN A placeholder layer for missing layers in a pipeline parallel model. cstdSr9)r[r\rr^r'r(r\]rzPPMissingLayer.__init__cOs|r|dStt|S)z>Return the first arg from args or the first value from kwargs.r)nextiterrBrr'r'r(forward`szPPMissingLayer.forward)rIrJrKrLr\rrr'r'r^r(rXs r max_bytescCs da|adSrd)_CPU_OFFLOAD_BYTES_CPU_OFFLOAD_MAX_BYTES)rr'r'r(set_cpu_offload_max_bytesisrc std}dur S|jtdkrSttkrSt}t}|s+Jdd}d}D]?}ttkr;n8tj|j |j |j j |j j d|d}||j |s\||_ n||_t||_ t|j |j 7ad}q3|r|sjfdd_S)Ncpuz3V1 CPU offloading requires uva (pin memory) supportTF)sizestriderlayoutrrcs:_fddD}t|||d}_|S)Ncs i|] \}}||jddqS)T)r)r)r;rv)rr'r(rGsz9maybe_offload_to_cpu..forward..)rr)rrr,r )rr device_stateoutputrrrZoriginal_forwardr'r(rs  z%maybe_offload_to_cpu..forward)r parametersrrRrrrr empty_stridedr=rr rr!copy__vllm_offloaded_cpu_datarnumel element_sizer)rZparamsr uva_availableuva_offloadingoffloaded_parametersrxcpu_datar'r%r(maybe_offload_to_cpuosF     r2num_hidden_layerslayer_fncsddlm}ddlm}|||j|j\}}tjddt |Dfddt ||Dddt ||D}|||fS)zgMake a list of layers with the given layer function, taking pipeline parallelism into account. r) get_pp_group)get_pp_indicescSg|]}tqSr'rrr'r'r(rCzmake_layers..cs$g|]}td|dqS)r`)r4)r2)r;idxr4r4r'r(rCscSr7r'r8rr'r'r(rCr9) vllm.distributed.parallel_stater5vllm.distributed.utilsr6 rank_in_group world_sizerRr ModuleListr)r3r4r4r5r6 start_layer end_layermodulesr'r;r( make_layerss    rD _model_to_pp_missing_layer_namesmodelcCsVt|}|tvr t|Sg}|D]\}}t|ttfr$||dq|t|<|S)zAGet the names of the missing layers in a pipeline parallel model.r`)idrErrrrr)rFmodel_idmissing_layer_namesr<rZr'r'r(get_pp_missing_layer_namessrJr<cs,t|ttfr dStfddt|DS)z=Check if a parameter is missing in a pipeline parallel model.Tc3rur9rv)r;missing_layer_namer<r'r(r@s  z*is_pp_missing_parameter..)rrrr{rJ)r<rFr'rLr(is_pp_missing_parameters  rMkeys hidden_sizecs(dtdtjdtjdtffdd }|S)N batch_sizerrr%cstfddDS)Ncs"i|] }|tjfdqS)rr)rRzeros)r;r*)rPrrrOr'r(rGszdmake_empty_intermediate_tensors_factory..make_empty_intermediate_tensors..r)rPrrrOrN)rPrrr(make_empty_intermediate_tensorss zPmake_empty_intermediate_tensors_factory..make_empty_intermediate_tensors)intrRrrr)rNrOrTr'rSr('make_empty_intermediate_tensors_factorys rVcCs|s|S|d|S)aAdd a prefix to a name if the prefix is non-empty. Args: prefix: The prefix to add. If empty, no prefix will be added. name: The name to potentially prefix. Returns: The string "prefix.name" if prefix was non-empty, otherwise just "name". r`r')r4r<r'r'r( maybe_prefixs rWcCs"|jj}|j}|rt||SdS)aeGet quantization config for Draft models. Draft models should use their own quantization config instead of the verifier/target model's config. This helper retrieves the draft model's quantization config. Args: vllm_config: The vLLM configuration object. Returns: The draft model's config if available, None otherwise. N)speculative_configdraft_model_config load_configrget_quantization_config)rrYdraft_load_configr'r'r(get_draft_quant_config s r]r+ layer_namenum_attn_modulec Cs|d}g}|D]}z |t|Wq tyYq w|dks&d|vr8t|dks4Jd|d|dSt|dksFJd|dt|dkrX|d||d}|S|d}|S) z Extract the layer index from the module name. Examples: - "encoder.layers.0" -> 0 - "encoder.layers.1.self_attn" -> 1 - "2.self_attn" -> 2 - "model.encoder.layers.0.sub.1" -> ValueError if num_attn_module == 1 r`r+attnz layer name z should only contain one integerrz! should contain most two integers)rarrUrrj)r^r_subnamesint_valssubname layer_indexr'r'r(extract_layer_index!s.     rftensorsoffsetcCs@|s |rt|jj|}tj|| |d}|S)N)minr)isinfr{isnanrRfinforrclamp)rhri clamp_valuer'r'r(cast_overflow_tensorsCsrprBtopkrcCs(|dkr tj||ddStj|||dS)a! Optimized topk implementation that uses torch.max for k=1 case. This function provides better performance for the common case of k=1 by using torch.max instead of the more general torch.topk. Args: values: Input tensor to find top-k values from topk: Number of top values to return (k). Must be > 0. dim: Dimension along which to compute topk Returns: Tuple of (values, indices) where values are the top-k values and indices are their corresponding indices in the input tensor r+T)rkeepdim)r)rRrrq)rBrqrr'r'r( fast_topkMsrscCstjj|Sr9)rRopsvllmsequence_parallel_chunk_implrer'r'r(sequence_parallel_chunkkrrwc Cspt}t}|d}||}|dkr#||}tj|ddd|f}n|}|jd|}||}t|d||Srd) rrrr functionalpadrrRnarrow) rftp_sizetp_rankseq_len remainderpad_lenychunkstartr'r'r(rvos rvcCs@t}t|d|}t|j}||d<tj||j|jd}|S)NrrQ) rrrrTrrRemptyrr)rfr{r}routr'r'r(!sequence_parallel_chunk_impl_fakes  r)op_nameop_func fake_impltagscCs0t|sdSd|vr d|_d|vrd|_dSdS)a% Update EAGLE model flags based on loaded weight name. This should be called during weight loading to detect if a model has its own lm_head or embed_tokens weight. Args: model: The model instance (must support EAGLE) name: The name of the weight to process Nlm_headT embed_tokens)rhas_own_lm_headhas_own_embed_tokens)rFr<r'r'r(process_eagle_weights  rfeature_layer_indexcCs|dkr ||dS|S)aGiven a signed vision feature layer, get the number of hidden layers needed to leverage it. Args: feature_layer_index: Index of a required layer in the visual encoder. num_hidden_layers: The total number of hidden layers in the visual encoder. rr+r')rr3r'r'r(get_layer_indexs r)r+)rg)frmcollections.abcrrr contextlibr dataclassesrrtypingrr r r rRtorch.nnr torch.funcr torch.nn.modules.moduler transformersr vllm.configrvllm.distributedrr vllm.loggerr3vllm.model_executor.layers.quantization.base_configr'vllm.model_executor.model_loader.reloadr-vllm.model_executor.model_loader.weight_utilsr%vllm.model_executor.models.interfacesrvllm.multimodalr vllm.sequencervllm.utils.math_utilsrvllm.utils.platform_utilsrrvllm.utils.torch_utilsrrrIrrPrNrrUrTrrrSrrrrrUrrrrtyperQrrrIdentityrrrrr2r@rDrErMrOrJrMrVrWr]rffloatrprsrwrvrTagneeds_fixed_stride_orderrrr'r'r'r(sP               ?n         -    ) C    $