o ۾i<+@sdZddlmZddlZddlmZddlmZddlm Z m Z m Z ddl m Z ddlmZddlmZdd lmZmZdd lmZdd lmZdd lmZd dlmZGdddejZGdddejZGdddejZ dS)zInference-only MiMo-MTP model.)IterableN)PretrainedConfig) CacheConfig ModelConfig VllmConfig)RMSNorm)LogitsProcessor)QuantizationConfig)ParallelLMHeadVocabParallelEmbedding)default_weight_loader)Qwen2DecoderLayer)IntermediateTensors) maybe_prefixc sneZdZ  ddededededBdedBddf fdd Z dd e j d e j d e j de de j f ddZ Z S)MiMoMultiTokenPredictorLayerNconfigprefix model_config cache_config quant_configreturncsptt|j|jd|_t|j|jd|_tj|jd|jdd|_ t ||||d|_ t|j|jd|_ dS)N)epsF)bias)rrrr) super__init__r hidden_size rms_norm_epstoken_layernormhidden_layernormnnLinear input_projr mtp_blockfinal_layernorm)selfrrrrr __class__W/home/ubuntu/.local/lib/python3.10/site-packages/vllm/model_executor/models/mimo_mtp.pyr-s z%MiMoMultiTokenPredictorLayer.__init__r inputs_embeds positionsprevious_hidden_statesspec_step_indexcCsj|dusJd||dk<||}||}|tj||gdd}|j||dd\}}||}||S)Nr)dim)r, hidden_statesresidual)rr r#torchcatr$r%)r&r+r,r-r.r1r2r)r)r*forwardDs      z$MiMoMultiTokenPredictorLayer.forward)NNr)__name__ __module__ __qualname__rstrrrr rr3Tensorintr5 __classcell__r)r)r'r*r,s8rc seZdZdddedeffddZdejdejfd d Z ddejd ejdejdejd Bde dejf ddZ ddejde de dejfddZ Z S)MiMoMultiTokenPredictorr vllm_configrcsrtjjj|_j|_tj j |_ t j fddt|j|j|jD|_tj |_dS)Nc s4i|]}t|td|jjjdqS)z.layers.)rrr)r:rrrr).0idxrrrAr)r* js z4MiMoMultiTokenPredictor.__init__..)rrr hf_confignum_hidden_layersmtp_start_layer_idxnum_nextn_predict_layersnum_mtp_layersr vocab_sizer embed_tokensr3r! ModuleDictrange mtp_layersrlogits_processorr&rArr'rDr*r]s"  z MiMoMultiTokenPredictor.__init__ input_idsrcCs ||SN)rLr&rRr)r)r*embed_input_ids{s z'MiMoMultiTokenPredictor.embed_input_idsNrr,r-r+ spec_step_idxcCs0|dur ||}|jt|j|||||SrS)rLrOr:rH)r&rRr,r-r+rVr)r)r*r5~s zMiMoMultiTokenPredictor.forwardr1lm_headcCs$|jt|j||||}|SrS)rOr:rHrP)r&r1rWrVlogitsr)r)r*compute_logitss z&MiMoMultiTokenPredictor.compute_logits)Nrr6)r7r8r9rr:rr3r;rUr<r5r rYr=r)r)r'r*r>\s8 r>cseZdZdddedeffddZdejdejfd d Z ddejd Bd ejdejde d Bdejd Bde dejfddZ d dejde dejd BfddZ de eeejfdeefddZdedefddZde dedefddZZS)!MiMoMTPr?r@rArcsJt|jj|_t|t|dd|_t|jj |jj t|dd|_ dS)Nmodel)rArrWr@) rrrrFrr>rr[r rKrrWrQr'r)r*rs    zMiMoMTP.__init__rRrcCs |j|SrS)r[rUrTr)r)r*rUs zMiMoMTP.embed_input_idsNrr,r1intermediate_tensorsr+rVcCs&|dksJd||||||}|S)Nrz+mimo_mtp only support predict one token now)r[)r&rRr,r1r\r+rVr)r)r*r5s   zMiMoMTP.forwardcCs|j||j|SrS)r[rYrW)r&r1rVr)r)r*rYszMiMoMTP.compute_logitsweightsc Cs gd}t|}t}|D]s\}}d|vrq||}|D]7\}}} ||vr)qd|vr/nNd|vr8||vr8q|||}|drH||vrHq||} | j} | | || n&|dra||vraqd|vrnd|vrnd|vrnq||} t| dt} | | || |q|S) N))qkv_projq_projq)r^k_projk)r^v_projv) gate_up_proj gate_projr)reup_projrzrotary_emb.inv_freqrOz mlp.experts.z.biasrLrW weight_loader) dictnamed_parametersset map_model_name_to_mtp_param_namereplaceendswithrhgetattrr add) r&r]stacked_params_mapping params_dict loaded_paramsname loaded_weight param_name weight_nameshard_idparamrhr)r)r* load_weightss>        zMiMoMTP.load_weightsrtc Csddl}d}|||}|r+t|d}||jj}|||d|d}gd}|D] }||vr;|Sq1d}|||}|rR|||d}|S) Nrz(model\.mtp_layers\.)(\d+)(\.)rr.)rr r#r%z(model\.mtp_layers\.\d+\.)z mtp_block.)regexmatchr<grouprrGrm) r&rtrepatternr} original_numnew_numname_without_prefixsub_namer)r)r*rls"    z(MiMoMTP.map_model_name_to_mtp_param_name spec_layercCsJgd}d}|D] }||vrd}nq|s#|d|dd|d}|S)z Rewrite the weight name to match the format of the original model. Add .mtp_block for modules in transformer layer block for spec layer )rLenormhnormeh_proj shared_headFTz model.layers.r{z .mtp_block.)rm)r&rrtspec_layer_weight_namesspec_layer_weightrwr)r)r*_rewrite_spec_layer_namesz MiMoMTP._rewrite_spec_layer_name)NNrr6)r7r8r9rr:rr3r;rUrr<r5rYrtuplerkrzrlrr=r)r)r'r*rZs@   $5rZ)!__doc__collections.abcrr3torch.nnr! transformersr vllm.configrrr$vllm.model_executor.layers.layernormr+vllm.model_executor.layers.logits_processorr'vllm.model_executor.layers.quantizationr 3vllm.model_executor.layers.vocab_parallel_embeddingr r -vllm.model_executor.model_loader.weight_utilsr vllm.model_executor.models.qwen2r vllm.sequencerutilsrModulerr>rZr)r)r)r*s"          0>