o ie,@sddlZddlmZddlmZddlmZddlZddl Z ddl m Z ddl m Z mZmZddlmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddl m!Z!ddl"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+m,Z,ddl-m.Z.ddl/m0Z0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ddlm?Z?m@Z@mAZAmBZBmCZCmDZDddlEmFZFddlGmHZHddlImJZJmKZKeeLZMGd d!d!ZNGd"d#d#eNZOd$e jPd%e9d&eQe jPe jPffd'd(ZRdS))N)replace) find_spec)cast) CUDAGraphMode VllmConfigget_layers_from_vllm_config) get_pp_group)set_forward_context) init_logger)AttentionLayerBase) get_model)supports_multimodal)DeepseekV32IndexerCache)SupportsMultiModal)Eagle3LlamaForCausalLM)MULTIMODAL_REGISTRY)current_platform)triton)is_pin_memory_available)AttentionMetadataBuilderCommonAttentionMetadata)AttentionBackendEnum)TreeAttentionMetadataTreeAttentionMetadataBuilder)TritonAttentionMetadata)CudagraphDispatcher) KVCacheConfig)SamplingMetadata) _SAMPLING_EPS)SpecDecodeMetadata)PADDING_SLOT_IDcompute_new_slot_mapping#copy_and_expand_eagle_inputs_kernel"eagle_prepare_inputs_padded_kernel&eagle_prepare_next_token_padded_kernelextend_all_queries_by_N) CpuGpuBuffer)coordinate_batch_across_dp)CachedRequestState InputBatchc@seZdZ d`dedejdefddZddZd d Z d d Z d dZ de fddZ de dejfddZ d`de dejdBdeeejffddZdeddfddZ   dadejdejdejdejd ejdBd!ed"ed#eeejejfdBd$ejdBd%eeejfeeeejfBdBdejfd&d'Zdejdejdejdejd ejdBd(ed$ejdBdee ejeffd)d*Zdefd+d,Zd-eee d.eeefd/ed0eee fdejf d1d2Zd!ed-ejd.eeefd/ed3ejdeejejff d4d5Zd!ed6e d7ejdeeejejffd8d9Z! d`d:e d;ejdejdZ"d!ed-eee d?ee deeejffd@dAZ#dBe$j%defdCdDZ&de$j%fdEdFZ'dGe$j%ddfdHdIZ(dJe$j%ddfdKdLZ)dJe$j%ddfdMdNZ*e+ O P dbde dQedRed%eeejfdBddf dSdTZ,de-fdUdVZ.defdWdXZ/dYe0ddfdZd[Z1d\e d]e dee ejffd^d_Z2dS)cSpecDecodeBaseProposerN vllm_configdevicepass_hidden_states_to_modelcCsX||_|jdus J|j|_|jj|_|jj|_||_||_||_|jj|_|jj |_ |j j |_ |jj |_ |j|_|j|_|jj|_|jsKdn|j |_|j|jrVdnd|_|jdk|_d|_d|_|jrm||jj}|jj|j||_t|j|_t |_!|j!"|j|_#d|_$d|_%g|_&g|_'|(|_)|jj*|_*t+|j|_,t-j.|jt-j/|d|_0|jj1|_1|jjj2|_2|jj2|_3|j1rt-j.d|jdft-j4|d|_5n&|j2dkr|j3dkrt-j.|j2|jdft-j4|d|_6n t-j.|jt-j4|d|_7t-j.|j|jf|j|d|_8t9|d|j}t-j||t-j/d|_|jr-|:|;|<d|_=d|_>|jrOt-j.|jft-j?|d|_=t-j.|jft-j?|d|_>t-j.|j|jf|j|d|_@tA|t-j/tB|dd|_Ct-j.|jt-j4|d|_Dd|_EtFGrddlHmI}tJ|g}tKtLjMjNd d rdd lOmP} |Q| dd lRmS} |Q| dd lTmU} |Q| tV||_E|jjW} | dusJtXY| |_Zt[|jZd} dg| }|jZD]}|t[|dd7<q|dg|_\|dg|_]t^d| D]}|j\Q|j\d|||j]Q||||dqt-jdt[|jZd|t-j/d_|d|_`dS)Nrdtyper,r,r0T)r0 pin_memoryr, with_numpy)RocmAttentionMetadataF)include_classname)AiterFlashAttentionMetadata)MLACommonMetadata)FlexAttentionMetadata)ar+speculative_configdraft_model_configmethodr-runnerr, model_configr0 max_model_lenparallel_configdata_parallel_rankdp_ranknum_speculative_tokensget_hidden_size hidden_sizeget_inputs_embeds_sizeinputs_embeds_sizeparallel_draftingextra_slots_per_requestnet_num_new_slots_per_requestneeds_extra_input_slotsparallel_drafting_token_id%parallel_drafting_hidden_state_tensor_init_parallel_drafting_paramsscheduler_config max_num_seqsmax_num_batched_tokensmax_num_tokensnparangetoken_arange_npr mm_registrysupports_multimodal_inputssupports_mm_inputsattn_metadata_builderdraft_indexer_metadata_builderattn_layer_namesindexer_layer_names,_get_eagle3_use_aux_hidden_state_from_configeagle3_use_aux_hidden_statecompilation_configrcudagraph_dispatchertorchzerosint32 input_ids uses_mropeuses_xdrope_dimdraft_uses_xdrope_dimint64mrope_positionsxdrope_positions positions hidden_statesmax'_raise_if_padded_drafter_batch_disabled_raise_if_multimodal_raise_if_mropeis_rejected_token_maskis_masked_token_maskbool inputs_embedsr&rbackup_next_token_ids_slot_mapping_bufferallowed_attn_typesris_rocm$vllm.v1.attention.backends.rocm_attnr5rrr ROCM_AITER_FAget_path(vllm.v1.attention.backends.rocm_aiter_far7append2vllm.model_executor.layers.attention.mla_attentionr8)vllm.v1.attention.backends.flex_attentionr9tuplespeculative_token_treeast literal_eval tree_choiceslencu_drafts_per_levelchild_drafts_per_levelrangerepeattree_draft_pos_offsets)selfr+r,r-r>max_batch_sizemax_num_slots_for_aranger5 rocm_typesr7r8r9spec_token_tree tree_depthnum_drafts_per_levelnodelevelrO/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm/v1/spec_decode/eagle.py__init__=s                                         zSpecDecodeBaseProposer.__init__cC|jjrtddS)NzSpeculative Decoding with draft models or parallel drafting only supports padded drafter batch. Please unset disable_padded_drafter_batch in the speculative_config.)r;disable_padded_drafter_batchNotImplementedErrorrrrrro z>SpecDecodeBaseProposer._raise_if_padded_drafter_batch_disabledcCs|jrtddS)NzbSpeculative Decoding with draft models or parallel drafting does not support multimodal models yet)rYrrrrrrps z+SpecDecodeBaseProposer._raise_if_multimodalcCr)NzWSpeculative Decoding with draft models or parallel drafting does not support M-RoPE yet)r<rfrrrrrrq$rz&SpecDecodeBaseProposer._raise_if_mropecCs^|jj}t|dr|j|_nt|dr|j|_ntd|jr-tj |j |j |j d|_ dSdS)N pard_token ptd_token_idztFor parallel drafting, the draft model config must have `pard_token` or `ptd_token_id` specified in its config.json.r/)r< hf_confighasattrrrMr ValueErrorr-rbemptyrFr0r,rN)rmodel_hf_configrrrrO+s      z5SpecDecodeBaseProposer._init_parallel_drafting_params num_tokenscCsT|jr|jddd|fS|jdkr#|jdkr#|jddd|fS|jd|SNr)rfrjrgrhrkrl)rrrrr_get_positions@s z%SpecDecodeBaseProposer._get_positionsrlcCsr|jr||jddd|f<dS|jdkr'|jdkr'||jddd|f<dS|jjjr0|d}||jd|<dSr)rfrjrgrhrkr+r?rl)rrrlrrr_set_positionsGs z%SpecDecodeBaseProposer._set_positions slot_mappingreturncsj|dur!|jd}|jd||||kr!|j||t|jd|fdd|j|jDS)zzReturn slot_mapping dict for EAGLE layers. If slot_mapping is provided, copies it into the buffer first. Nrcsi|]}|qSrr).0nameviewrr dsz.)shaperwcopy_fill_r r\r])rrr num_actualrrr_get_slot_mappingTs z(SpecDecodeBaseProposer._get_slot_mappingcudagraph_modecCs:|jjs|tjtjfvrtj}ntj}|j|dS)zInitialize cudagraph dispatcher keys for eagle. Eagle only supports PIECEWISE cudagraphs (via mixed_mode). This should be called after adjust_cudagraph_sizes_for_spec_decode. N) r; enforce_eager mixed_moder PIECEWISEFULLNONErainitialize_cudagraph_keys)rreagle_cudagraph_moderrrrfs z0SpecDecodeBaseProposer.initialize_cudagraph_keystarget_token_idstarget_positionstarget_hidden_statesnext_token_idstoken_indices_to_samplecommon_attn_metadatasampling_metadatamm_embed_inputsnum_rejected_tokens_gpu slot_mappingsc - Cs|} |jdkr!t|jtsJ|j|}|jd|jks!J|j||||||| d\} }}|j dus7J|j durA| } n|j } | j |dd}|j rW|j j |dd}nd}i}|jD]}|||<q^|jD] }|duspJ|||<qh|j| | d\}}|j|\}}|j}|dur|||j<|jr|pd\}}|jj|jd| ||d|jd| <d}|jd|}n |jd|}d}||||d }|jr|jd||d <t||j||||||jd |jdi|}| s|}|}n|\}}Wdn 1swY||}|j!|} |j"d ks |j#r-| j$dd }!|!%d|j"S|j&r;|j'dd|f}"n|j(|}"|jdvrL|j|}n||}t|t)rh|j*| | |"||| d}#t+j,|#d d S| j$dd }!|j-durt||j-st.dt/|d|j-|!g}#|j| | d\}$}%|j|$\}}|j}&|%dur|&|%|j<| |_0d |_1|j2d| d |_3t+4|j5d| d 6|_7|j"d kr| dur|j8| 8_8d|_9d|_:t;|j"d D]^}'|#d<}|j&r |"d 7}"|"d|j=k}(t+>|(?dt+@|"|"})n|"d 7}"|"|j=k}(t+>|(d|"})|j8d 7_8|j8A|(d tB|jCd |j=|_C|j9durB|j9d 7_9|j:durO|j:d 7_:| jDjE}*|j&r^|)d|*}+n|)|*}+|jFjGd |+%dd d},|,%d},|j&r|,|*|)d|*|_n |,|*|)|*|_|jA|(tH| j ||'d d}|jD]}|||<q||jd| <|I| |)||jd| <|jr|j||jd| <d}|jd|&}n |jd|&}d}|||&|d }|jr|jd|&|d <t||j|&|%|||&|jd |jdi|}| s|}|}n|\}}Wdn 1s%wY|d| }|j!|d| } | j$dd }!|#J|!qt+jK|#d d }!|!S)Neagle3r:)rrrrrcadrrr draft_indexnum_tokens_unpaddednum_tokens_padded)NN)multimodal_embeddings is_multimodalrerlrurmrnum_tokens_across_dpcudagraph_runtime_moderr.dim) deepseek_mtp ernie_mtplongcat_flash_mtppangu_ultra_moe_mtp) batch_sizelogitsrlrmrrz^Unsupported attention metadata type for speculative decoding with num_speculative_tokens > 1: z. Supported types are: rindexr)Lrr= isinstancemodelrcombine_hidden_statesrrFset_inputs_first_passr>rZ_get_attention_metadata_builderbuild_for_draftingr[r\r]_pad_batch_across_dpradispatchrrCrYembed_input_idsrerurr-rmr r+rrmodel_returns_tuplecompute_logitsrDrIargmaxrrfrjrlr propose_treerbcatrxrtypenum_actual_tokens max_query_lenrUquery_start_loc from_numpyrVclonequery_start_loc_cpuseq_lens _seq_lens_cpu_num_computed_tokens_cpurintr@where unsqueeze zeros_like masked_fill_min max_seq_len kv_cache_spec block_sizeblock_table_tensorgatherr rr~stack)-rrrrrrrrrrrrrrZ attn_metadatadraft_indexer_metadataper_layer_attn_metadata layer_namenum_tokens_dp_paddedrr batch_descnum_input_tokens mm_embeds is_mm_embedreru model_kwargsret_hidden_stateslast_hidden_statesrmsample_hidden_statesrdraft_token_idsrldraft_token_ids_listbatch_size_dp_paddedbatch_size_across_dpinput_batch_size token_indexexceeds_max_model_lenclamped_positionsr block_numbers block_idsrrrproposews                                     zSpecDecodeBaseProposer.proposercCs|jsG|dur|jddd}|jd}|dd|jd|d<||j|<|jdkr5|jdkr5|d}|||||jd|<|||fS|jdusNJ|j dusUJ| } |j |j } t dt| } | | d| } |jd} | |j | }tj| |jtj|jd}tj| tj|jd}| | f}|j}|jddd}|dur||}t|did|d|d|d|jd |jd |jd |j d |d |d|d|ddd|jd| d|jd|jd| |jr|jdusJ||j|<|j d|}tj|d|j|jd||jd|d|jdur#|n|j}t||jd||jd||jj |j |j!d}t"||j |j#|d}|||fS)Nr.rr/target_token_ids_ptrtarget_positions_ptrnext_token_ids_ptrout_input_ids_ptrout_positions_ptrout_is_rejected_token_mask_ptrout_is_masked_token_mask_ptrout_new_token_indices_ptrout_hidden_state_mapping_ptrquery_start_loc_ptrquery_end_loc_ptrpadding_token_idrMtotal_input_tokensnum_padding_slots_per_requestshift_input_idsBLOCK_SIZE_TOKENSout)r new_positionsrrrnum_new_tokensr@)NrUnew_slot_mappingr)$rLrrrergrhrrmrrrsrrrKrrnext_power_of_2rbrrJrdr,r"rlrMr-rNrrrZrr!rrr@r%rU)rrrrrrrrrrmax_num_tokens_per_requestr) num_blockstotal_num_input_tokenstotal_num_output_tokensout_hidden_state_mappinggridr query_end_locmaskbuilderr/new_cadrrrrs                      z,SpecDecodeBaseProposer.set_inputs_first_passcCs |jdvS)N)mtp draft_model)r=rrrrrKs z*SpecDecodeBaseProposer.model_returns_tuplesampled_token_idsrequestsgpu_input_batchnum_scheduled_tokensc Csv|j}g}t|D]$\}}|r|d} n||} || } | j|| } | | } || q tj|tj|jj d}|S)aj This function is used to prepare the inputs for speculative decoding. It calculates the next token ids for each request based on the sampled token ids from the CPU. If a request has no sampled token ids (e.g., during the initial decoding steps), it falls back to using the request state to get the next token id. r:r/) req_ids enumeratenum_computed_tokens get_token_idr~rbtensorrdrer,) rr=r>r?r@rAri token_ids next_token_idreq_id req_stateseq_lenrrrprepare_next_token_ids_cpuNs    z1SpecDecodeBaseProposer.prepare_next_token_ids_cpudiscard_request_maskc sj}tjfddt|Dtjd|jjd|<|j||jj}|j\}} |j } |j t j ks6J|j t jks>Jt j |t j| d} | |} |f} t| }t| |||| | j| ||d|d | | fS)a This function is used to prepare the inputs for speculative decoding. It calculates the next token ids and the number of valid sampled tokens for each request, considering the "discarded" requests whose next token is not sampled and comes from `request.get_token_id()` instead. This is denoted the "backup" token id. It also counts rejected tokens via `sampled_token_ids`. cs*g|]}j|j|qSr)rArD seq_lens_cpuitem)rrFrr?r>rr s  zHSpecDecodeBaseProposer.prepare_next_token_ids_padded..r0Nr/r)r))num_reqsrTarrayrrdrv copy_to_gpugpurr,r0rbrtr new_emptyrr0r$ vocab_sizestride)rrr=r>r?rMrSbackup_tokens_gpurrr,rvalid_sampled_tokens_countr6r)rrPrprepare_next_token_ids_paddedos<     z4SpecDecodeBaseProposer.prepare_next_token_ids_paddedspec_decode_metadatar[c Cs|j}|j}tj|ftj|d}tj|ftj|d}|f}t||j||j||||j} | dd| dd} | d } t |j|j | |j |j |j| |  |j |j|jd| d|jd } | ||fS)a This function is used to prepare the inputs for speculative decoding It updates the common_attn_metadata for speculative decoding, but does not consider the rejected tokens. Instead, all tokens are included as inputs to the speculator, with the rejected tokens used as padding and filtered out later by `token_indices_to_sample`. No blocking CPU operations should be introduced in this function. r/r.Nr:T rrrrrrSrrrrrcausaldcp_local_seq_lens)rSr,rbrrdr#cu_num_draft_tokensrrrOrrrrrnrNrrr`) rrr]r[rSr,rrr6rnew_query_len_per_reqtotal_num_tokensspec_common_attn_metadatarrrprepare_inputs_paddedsN       z,SpecDecodeBaseProposer.prepare_inputs_paddedrrrmc& Cs&|jjdd}t|tsJ|jd}|} |jd} | dkr,|jdd|d} n t j || ddj |d} | g} ||dd} t j d|j j|j jd}t j d|jj|jjd}t j d|jj|jjd}||d|jd|ddf}t|j}t|dD]}||d}|||jk}t |d||d}| dkr|j| dd}| dkr| j| dd} t j|| gdd}t j||gdd}t j|| gdd}|}t|||jd|d|j| |||d}|j||dd}i}|jD]}|||<qt|j|j|_|j |d|j!j"}|dd|||f}||}|j#j$d|d}||||}t%||<|d|_&|j'}|d} | |j d|<|d|jd|<||d|jd|<|j()|\}!}"|"j*}#t+||j,|#|!|-|#|j&d "|j.|j d|#|jd|#|jd|#dd \}$}Wdn 1swY|d|||ddd| df} |$d|||ddd| df}%|j./|%0|| d}|j|d} | dkr|jdd|d} n t j || ddj |d} | 1| |j|d|} |j|d}q| S) Nrr.r:rr2)rrrrrr)rrr)rerlrmru)2r> attn_groupsget_metadata_builderrrrrrrrbtopkindicesrrer,r0rlrmrrrr@rrepeat_interleaverrrUrrr\rrrrr block_tablerr rrrarrr r+rrrreshaper~)&rrrrlrmrrtree_attn_metadata_buildertotal_num_draftslevel_num_drafts num_childrenrrdraft_hidden_statestree_input_idstree_positionstree_hidden_statesflattened_draft_positionsrrdraft_positionsr query_lenrrrrquery_positionsrrrrrerrrr draft_last_hidden_statesrrrrs                  z#SpecDecodeBaseProposer.propose_treenum_draft_tokenscsXfddt|D}tj|tjd}|jj}|j}|j|}|dd|dd}||} | } tj |j tjt d} | } t j | | ddd| d} t | dd| }|jd| |}t |dd| }||}t|j|d d }t| j|d d |j|d d | ||j|j| |||j|j|d |jd }||fS) a+ This function is used to prepare the inputs for speculative decoding. It updates to the common_attn_metadata to account for the rejected tokens (and newly sampled tokens). It also returns the token indices of the tokens that should be fed to the speculator. cs0g|]\}}|dkr|dt|ndqS)rr.)r)rrFnr=rrrQsz9SpecDecodeBaseProposer.prepare_inputs..rRr.Nr:)r0r3r*T) non_blockingr^)rBrbrErdrr,rrNnumpyrcrrrTcumsumrrVrtorrrSrnrOrrr`)rrr=rznum_rejected_tokensr,rnew_seq_lens_cpurbnew_num_tokens_per_reqnew_num_tokens_per_req_npnew_query_start_loc_cpunew_query_start_loc_nprcnew_query_start_locs_expanded token_offsetsold_query_start_locs_expandedtoken_indices_np token_indicesrdrr|rprepare_inputssX       z%SpecDecodeBaseProposer.prepare_inputsrcCst|dr|j}|jjS)Nmodule)rr __class____name__)rrrrrget_model_names z%SpecDecodeBaseProposer.get_model_namecCsLddlm}|dt|j|jjd}Wd|S1swY|S)z Default method to call get_model(). Can be overridden by subclasses which need to customize model loading. r) set_model_tag eagle_head)r+r?N)vllm.compilation.backendsrr r+r;r<)rrrrrr _get_models   z!SpecDecodeBaseProposer._get_model target_modelc Cstt|jt}tt|jt}||_t|jt|}t|jt}||}t|||_ t||_ |j r[|j d}|| || |j|j |j|j|_nd|_|jrztjdgg|jjd}|jj|ddWntttfytdd|_Ynwt|rt|dsJ||dvr|jj|jj_n||d kr|jj j|jj_n|jj|jj_t!t"|#} n|} |$| |%| |j&r|j'r|j(dusJ|j()|j*r|j+|jj,-d |j.n|jj,-|j.dSdSdS) Nrr.)r,)rzNDraft model does not support multimodal inputs, falling back to text-only modeFconfig)"Qwen2_5_VLForConditionalGenerationQwen3VLForConditionalGeneration"Qwen3VLMoeForConditionalGeneration!HunYuanVLForConditionalGenerationGlmOcrForConditionalGenerationPixtralForConditionalGenerationr1)/setrr+r keysrrrlistr\r]get_attn_backendget_builder_clsget_kv_cache_specr,r[rYrbrErerrAttributeError TypeErrorloggerwarningr rrrimage_token_idimage_token_index vision_configrrget_language_model_maybe_share_embeddings_maybe_share_lm_headrIr-rNrr_r mask_hiddenrrF) rrtarget_attn_layer_namestarget_indexer_layer_namesdraft_attn_layer_namesindexer_layersdraft_indexer_layer_names first_layerdummy_input_idstarget_language_modelrrr load_model s          z!SpecDecodeBaseProposer.load_modelrcCs tjdkrt|dd}|durtdt|dr|j}n t|dr&|j}ntdd}t|jd rl|jjs>d }t d n5t |j t jrft |jjjj t jrft |j |jjjj rfd }t d n t d nd }t d|rt|jjdr|jj`||jj_dSdSt ddS)a Some draft models may not have their own embedding layers, and some may have a duplicate copy of the target model's embedding layers. In these cases, we share the target model's embedding layers with the draft model to save memory. r.rNz,Target model does not have 'model' attribute embed_tokens embeddingzBTarget model does not have 'embed_tokens' or 'embedding' attributeFhas_own_embed_tokensTzDetected EAGLE model without its own embed_tokens in the checkpoint. Sharing target model embedding weights with the draft model.zDetected EAGLE model with embed_tokens identical to the target model. Sharing target model embedding weights with the draft model.zrDetected EAGLE model with distinct embed_tokens weights. Keeping separate embedding weights from the target model.zPDetected MTP model. Sharing target model embedding weights with the draft model.zRThe draft model's vocab embedding will be loaded separately from the target model.)r world_sizegetattrrrrrrrrinforweightrbTensorequalcpu)rr inner_modeltarget_embed_tokensshare_embeddingsrrrrls\       z.SpecDecodeBaseProposer._maybe_share_embeddingscCsVd}t|jdrG|jjsd}tdn:t|drAt|jjtj rAt|jjjtj rAt |jj |jjj rAd}tdn tdnd}td|rt|drt|jdr^|j`|j|j_t |jd d }|rrt |d d nd }|d urt|t jr|n|}|D]"}t |d d }|d urt|d r|`|j|_tdqd Sd Sd Sd S)z Some draft models may not have their own LM head, and some may have a duplicate copy of the target model's LM head. In these cases, we share the target model's LM head with the draft model to save memory. Fhas_own_lm_headTzzDetected EAGLE model without its own lm_head in the checkpoint. Sharing target model lm_head weights with the draft model.lm_headz{Detected EAGLE model with lm_head identical to the target model. Sharing target model lm_head weights with the draft model.zkDetected EAGLE model with distinct lm_head weights. Keeping separate lm_head weights from the target model.zNDetected MTP model. Sharing target model lm_head weights with the draft model.rNlayers shared_headheadz6Shared target model lm_head with MTP shared_head.head.)rrrrrrrrrbrrrrnn ModuleDictvaluesr)rr share_lm_headinnerritemslayershrrrrsb        z+SpecDecodeBaseProposer._maybe_share_lm_headTFuse_cudagraphsis_graph_capturingc Cs<t|s|jndD]}|dkr4|j||d\}}|r&|j|\}} | j} ntj}|} |dur4| ||j<|j rH|durH|j d|vrH| | } n|pKi} t d|j | ||| d<|j red} |jd| } n |jd| } d} t| || | d}|jr|jd| |d<|jdi|Wdn1swYq dS)Nr.rrrrrmr)rrDrrarrrrrCr\rr r+rYruredictrr-rmr)rrrrrfwd_idxrrrrrslot_mapping_dictrerukwargsrrr dummy_runsZ     z SpecDecodeBaseProposer.dummy_runcCs\d}|jd}|jjD]}|D] }||jvr|}nq|dur#nq |dus,Jd|S)zFind and return the attention metadata builders for EAGLE layers. Returns: The metadata builders for EAGLE layers. Raises: AssertionError: If no metadata builders are found for EAGLE layers. Nrz;Failed to find attention metadata builder for EAGLE layers.)r\r>rf layer_namesrg)rr9 chosen_layerkv_cache_group attn_grouprrrr2s    z6SpecDecodeBaseProposer._get_attention_metadata_buildercCs:|jdkrdSd}t|jjdd}|dur|dd}|S)a5 Some eagle3 heads (e.g., nvidia/gpt-oss-120b-Eagle3-v2) do not use auxiliary hidden states and directly uses the last layer output just like eagle1. They might indicate this by setting "use_aux_hidden_state" to False inside the "eagle_config" dict of their hf_config. rFT eagle_configNuse_aux_hidden_state)r=rr<rget)rrrrrrr^Ks  zCSpecDecodeBaseProposer._get_eagle3_use_aux_hidden_state_from_configkv_cache_configcsXit|jD]\}}|jD]}||<qqttfdd|jDdks*JddS)z Validate that all drafting layers belong to the same KVCacheGroup. Need this assumption to ensure all drafting layers can use the same AttentionMetadata. May extend to multiple AttentionMetadata in the future. csg|]}|qSrr)rrkv_cache_groupsrrrQiszGSpecDecodeBaseProposer.validate_same_kv_cache_group..r.z)superr)rr+r,r>rrrrs  zEagleProposer.__init__r)rrrrrbr,r __classcell__rrrrrsrrrrcCs|jr|}|jdd}||fS|jdusJ|j}|js'|tk}t|d|}||dd|j dtj d}t |}| | |jddd}|js]|jdd}t|||}||fS)Nr:rg?r.)rr0) all_greedyr temperature all_randomrrbrdiv_rsoftmaxfloat32 empty_like exponential_div)rrprobsrr is_greedyqgreedy_token_idsrrr#compute_probs_and_sample_next_tokens$   r)Sr dataclassesrimportlib.utilrtypingrr~rTrbtorch.nnr vllm.configrrrvllm.distributed.parallel_statervllm.forward_contextr vllm.loggerr /vllm.model_executor.layers.attention_layer_baser vllm.model_executor.model_loaderr vllm.model_executor.modelsr &vllm.model_executor.models.deepseek_v2r%vllm.model_executor.models.interfacesr'vllm.model_executor.models.llama_eagle3rvllm.multimodalrvllm.platformsrvllm.triton_utilsrvllm.utils.platform_utilsrvllm.v1.attention.backendrr#vllm.v1.attention.backends.registryr$vllm.v1.attention.backends.tree_attnrr&vllm.v1.attention.backends.triton_attnrvllm.v1.cudagraph_dispatcherrvllm.v1.kv_cache_interfacervllm.v1.sample.metadatarvllm.v1.sample.samplerrvllm.v1.spec_decode.metadatarvllm.v1.spec_decode.utilsr r!r"r#r$r% vllm.v1.utilsr&vllm.v1.worker.dp_utilsr'vllm.v1.worker.gpu_input_batchr(r)rrr*rrrrrrrrsp                           Z