o ۾i3@sUddlZddlZddlZddlZddlZddlmZddlmZm Z m Z ddl m Z ddl m Z mZddlmZddlmZddlmZmZmZmZmZddlZddlZddlZddlmZdd lmZddlm Z dd l!m"Z"dd l#m$Z$m%Z%dd l&m'Z'dd l(m)Z)m*Z*m+Z+m,Z,m-Z-ddl.m/Z/m0Z0ddl1m2Z2ddl3m4Z4m5Z5ddl6m7Z7ddl8m9Z9m:Z:m;Z;mZ>ddl?m@Z@mAZAddlBmCZCddlDmEZEmFZFddlGmHZHmIZIddlJmKZKddlLmMZMddlNmOZOmPZPddlQmRZRmSZSddlTmUZUmVZVddlWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbddlcmdZdmeZemfZfddlgmhZhddlimjZjdd lkmlZlmmZmmnZndd!lompZpdd"lqmrZrdd#lsmtZtdd$lumvZvdd%lwmxZxmyZymzZzdd&l{m|Z|dd'l}m~Z~dd(lmZdd)lmZmZdd*lmZmZdd+lmZdd,lmZdd-lmZmZdd.lmZmZmZmZmZmZmZdd/lmZdd0lmZmZmZdd1lmZdd2lmZdd3lmZmZmZmZmZmZmZmZmZmZmZdd4lmZmZmZmZmZmZmZmZmZmZmZdd5lmZmZdd6lmZmZdd7lmZdd8lmZdd9lmZdd:lmZdd;lmZddlmZdd?lmZdd@lmZddAlmZmZddBlmZddClmZmZddDlmZddElmZddFlmZmZddGlmZddHlmZddIlmZddJlmZmZmZmZddKlmZddLlmZdMdNlmZmZmZmZerddOlmZddPlmZmZddQlmZeCeZeeefZeedR<eeeBZeedS<GdTdUdUeZGdVdWdWeZGdXdYdYeZGdZd[d[eee݃ZeGd\d]d]ZdS)^N) defaultdict)IterableIteratorSequence)contextmanager)copydeepcopy) dataclass)reduce) TYPE_CHECKINGAny NamedTuple TypeAliascast)tqdm)compilation_counter) CUDAGraphStatCUDAGraphWrapper)set_cudagraph_capturing_enabled)CompilationMode CUDAGraphMode VllmConfigget_layers_from_vllm_config update_config)get_ec_transferhas_ec_transfer) EplbState)get_kv_transfer_grouphas_kv_transfer_group)copy_kv_blocks) get_dcp_group get_pp_group get_tp_group graph_captureis_global_first_rank&prepare_communication_buffer_for_model)BatchDescriptorset_forward_context) init_logger) LoRAMappingLoRAMappingType) Attention MLAAttention)AttentionLayerBase)RoutedExpertsCapturer)MRotaryEmbeddingXDRotaryEmbedding)TensorizerLoaderget_model_loader)finalize_layerwise_reloadinitialize_layerwise_reload) MultiModalEmbeddings SupportsMRoPESupportsMultiModalSupportsXDRoPEis_mixture_of_expertssupports_eagle3supports_mropesupports_multimodal_pruningsupports_realtimesupports_transcriptionsupports_xdrope)VllmModelForPoolingis_pooling_modelis_text_generation_model)MULTIMODAL_REGISTRY)MultiModalBudget)BatchedTensorInputsMultiModalKwargsItemPlaceholderRange)group_mm_kwargs_by_modality) PoolingParams) SamplingType)IntermediateTensors)GenerationTask PoolingTask SupportedTask) instrument)&length_from_prompt_token_ids_or_embeds)json_map_leaves)cdivround_up)DeviceMemoryProfiler format_gib)PytHooks)is_pin_memory_available)get_dtype_sizekv_cache_dtype_str_to_dtype)AttentionBackendAttentionCGSupportAttentionMetadataAttentionMetadataBuilder AttentionTypeCommonAttentionMetadata MultipleOf)GDNAttentionMetadataBuilder)"create_fast_prefill_custom_backendget_dcp_local_seq_lens+reorder_batch_to_split_decodes_and_prefills)NewRequestData)CudagraphDispatcher) AttentionSpecChunkedLocalAttentionSpecCrossAttentionSpecEncoderOnlyAttentionSpecFullAttentionSpec KVCacheConfigKVCacheGroupSpec KVCacheSpec MambaSpecSlidingWindowSpecUniformTypeKVCacheSpecs) EMPTY_MODEL_RUNNER_OUTPUTAsyncModelRunnerOutput DraftTokenIdsECConnectorOutputKVConnectorOutput LogprobsListsLogprobsTensorsModelRunnerOutput PoolerOutput SamplerOutput&make_empty_encoder_model_runner_output)PoolingMetadata PoolingStates)LogitsProcessorsbuild_logitsprocs)LogitsProcessor)SamplingMetadata)RejectionSampler)Sampler)DraftModelProposer) EagleProposer)MedusaProposer)SpecDecodeMetadata)SuffixDecodingProposer)apply_grammar_bitmask) CpuGpuBufferrecord_function_or_nullcontext) mamba_utils) check_attention_cp_compatibilityget_total_cp_world_size)coordinate_batch_across_dp)ECConnectorModelRunnerMixin)CachedRequestState InputBatch) UBatchWrapper)KVConnectorModelRunnerMixin)LoRAModelRunnerMixin) UBatchSlicescheck_ubatch_thresholdsmaybe_create_ubatch_slicessplit_attn_metadata)is_residual_scattered_for_sp)lock_workspace)AttentionGroup(add_kv_sharing_layers_to_kv_cache_groups bind_kv_cachesanity_check_mm_encoder_outputs)TensorizerConfig) GrammarOutputSchedulerOutput NgramProposerAttnMetadataDictPerLayerAttnMetadatac @sJeZdZdedejdedBdeedej j def dd Z d efd d Z dS) AsyncGPUModelRunnerOutputmodel_runner_outputsampled_token_idslogprobs_tensorsNinvalid_req_indicesasync_output_copy_stream vocab_sizecCs||_||_t|_||_||_||_tj }tj |'| ||jj ddd|_ |jr6|jnd|_|jWddS1sIwYdSNcpuT non_blocking)_model_runner_output_invalid_req_indicestorchEventasync_copy_ready_event_sampled_token_idsr_logprobs_tensorscudacurrent_streamstream wait_streamtosampled_token_ids_cputo_cpu_nonblocking_logprobs_tensors_cpurecord)selfrrrrrrdefault_streamrS/home/ubuntu/.local/lib/python3.10/site-packages/vllm/v1/worker/gpu_model_runner.py__init__s$      "z"AsyncGPUModelRunnerOutput.__init__returncCs|jjd}|j|`|`|dkr1|j}|jD]}||qd}|j dur0|j }nt j |j|j |j|j d\}}|j}||_||_|S)zCopy the device tensors to the host and return a ModelRunnerOutput. This function blocks until the copy is finished. rNr)rshaper synchronizerrtolistrclearrtolistsr parse_outputrrrlogprobs)r max_gen_lenvalid_sampled_token_idsilogprobs_listsoutputrrr get_outputs,       z$AsyncGPUModelRunnerOutput.get_output) __name__ __module__ __qualname__ryrTensorrxlistintrStreamrrrrrrrs #rc@s<eZdZdededeedejj fddZ defdd Z d S) AsyncGPUPoolingModelRunnerOutputrraw_pooler_output finished_maskrcCs||_t|_||_tj}tj|'||t dd|j}|j ddt ||D|j_ WddS1s@wYdS)NcS|durdS|jdddSrrxrrr%z;AsyncGPUPoolingModelRunnerOutput.__init__..cSg|] \}}|r |ndqSNr.0outincluderrr ) z=AsyncGPUPoolingModelRunnerOutput.__init__..) rrrr_raw_pooler_outputrrrrrQrzip pooler_output)rrrrrrraw_pooler_output_cpurrrrs     "z)AsyncGPUPoolingModelRunnerOutput.__init__rcCs|j|`|jS)zCopy the device tensors to the host and return a ModelRunnerOutput. This function blocks until the copy is finished. )rrrrrrrrr.s z+AsyncGPUPoolingModelRunnerOutput.get_outputN) rrrryrzrboolrrrrrrrrrrs rc@seZdZUdZded<ejed<edBed<edBed<ejed<ejed <e ejdBed <e dBed <e dBed <e e ejfe e e ejfBdBed <dS)ExecuteModelStatezwEphemeral cached state transferred between execute_model() and sample_tokens(), after execute_model() returns None.rscheduler_outputlogitsNspec_decode_metadata spec_decode_common_attn_metadata hidden_statessample_hidden_statesaux_hidden_statesec_connector_outputcudagraph_stats slot_mappings)rrr__doc____annotations__rrrr_rrurdictstrrrrrr9s        ,rc@s eZdZdedejfddZdeddfdd Zd6d d Z d6d d Z e d6ddZ de fddZdddeejBdejdedefddZddZd7ddZd6d d!Zd6d"d#Zd7d$d%Zd&ejddddfd'd(Zd)ed*edefd+d,Zd-efd.d/Zd-efd0d1Z ddde!fd2d3Z"d4ede!fd5d6Z# d8de$j%d7e$jdBde&e$j%e$j%ffd8d9Z'ddd:ed;e$j%ddfdd9d?e)eefd@e*dAedBede&ejdBe$j%dBff dCdDZ+ddd?e$j%de&eje,dBffdEdFZ-     > >   d:dedAedGedHedBdIedBdJe.dBdKejdBdLedBed?e)eefdBdMe/e/edBdNe)eejfdBde&e0e1dBffdOdPZ2d?e$j%dQe$j%dRe/ede/e/edBfdSdTZ3d?e$j%dQe$j%dRed@e*dUe4def dVdWZ5d;dXdYZ6d;dZd[Z7d\e$j%d]e$j%de,fd^d_Z8dKejdejfd`daZ9ddde&e/ee/e&ee:fe/e&ee;fffdbdcZde?j@fdjdkZAde/eBfdldmZCde/eDfdndoZEde&eFdpffdqdrZGdedseHdBdtedeHfdudvZId=dwedxeddfdydzZJd{ejd?ed|e$j%d}eKdBdeLeMBf d~dZNd?edefddZOdede&ejdBejffddZP d8dddedseHdBde&ejdBejdBejeHdBe)ee feQdBffddZRdejdBde,dBdeSfddZTdddeSdejdBd{ejd?ede,dBde&e)eefeUdBe/e/ee)eeVdBfe/ee)eefe/effddZWeXddZY    d>dejdBdejdBdseHdBdejdBde)ee fde f ddZZe[ d8dedededAededBdef ddZ\  >    fd?dedAed|e$j%dedededededBdedBdedBdede&e]e^eejdBe_dBffddZ`d6ddZa d8dHedIededJdde&e)eejfdBe)eejfe/e)eejfBdBff ddZbe d8dddseHdBdeLeMBeHBdBfddZcej dddeLeMBeHBfddZddejddfddZed6ddZfdegdBfddZh >d9dddeddfddZide&e/e/ee/effddZjdejdejddfddZkde/efddZldddeje/e/eBdemd{ejdejde/ejdBde,dBde1dNe)eejfe/e)eejfBdBde/e/eejBfddƄZnde)ee fddfddɄZoepddˍd9deddfdd΄Zqde&edpfdBfddЄZr   d@dese&eejfdBdedBdeddfddՄZt  dAddلZud{ejd?e)eefde)eeVdBffddۄZvdejdBde)eeffdd݄ZweXdejdBdejdBfdd߄Zxdedede!fddZye  > >  > > >  > fdBdede]dBdededededxededededede&ejejffddZze d{ejdejfddZ{d{ejdeDde|fddZ}e d{ejde|fddZ~d6ddZepddˍdefddZde/e^de]fddZdeddfddZdede/eddfddZde/eeede/eddfddZd6ddZe[d ed e/edefd d Zdede/eddfd dZdede)eejffddZdeefddZdeefddZdede/efddZdede)eejfde/ede)eejffddZde)eejfddfddZdede/ede)eejffddZdeddfdd Zdeddfd!d"Zd#d$Zd%eddfd&d'Zd6d(d)Zde)ee*ffd*d+Zdejde/e/efd,d-Zde)ee)eeeBfffd.d/ZeXd0ed1e/e&ee fd2ed3efd4d5ZdS(CGPUModelRunner vllm_configdevicec Cs8||_|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j|_|j |_ ddl m }|t |jj d|j}|j}|j}|j}||_t|_|jj|_t|j|j|_|jdk|_|j|_|j|_d|_|j|_|jj|_|jj|_|jdkr}dntj|_ |j!|_"|j#|_$|jj%dkot&t'j(dk|_)|*||_+|,|_-|j.|_.|j/|_0|jj1 |_2|jj3|_3t4|_5|j6|_6|j7|_7|j58||_9|jj:r|j;|_t?|jj@d|_Ad|_B g|_Cd|_Dd|_Eg|_Fi|_Gd|_H|jrrt'jIrr||jjJd krdd lKmL}||j|_MnU|jNr(tO|j|j|d |_MnD|jjJd kr6tP|j|_Mn6|jQrRtR|j|j||_M|jjJd krQ|jMjS|_Hn|jjJdkrctT|j|jd|_Mn tUd|jjJtV|jA|_Wd|_X|jr|jjY|_X|jjZ} | dur| jdur| j|_[n|j|_[i|_\i|_]t^j_`|_a|jb} | durtc| nd} td|j$te|j|j<|j"|j|j|jf|jjgg|jjggth|jjti|j|j|j|j| th| |j|jjjd |_kd|_ld|_m|j>rt^j_`|_lt^n|_m|jjor|jjptqjrkrts|jjo|_t|ui|_vtwx|_y|jz|j"t^j{d|_||jz|j"t^j}d|_~|jz|j$dt^j{d|_|jz|j$t^j{d|_|jz|j$t^j{d|_|jdkr`|jz|j$t^j{d|_|jz|j"|j-|jdd|_|jz|j"t^jhd|_|jz|j$t^jhd|_|jz|j$t^j{d|_|jz|j$t^j}d|_|j9r|jz|j"t^jhd|jz|j"t^jhdg|_d|_|j6r|jzd|j"dft^j}d|_|j7dkr|jz|j7|j"dft^j}d|_d|_tjte|j$d|j|j"tj}d|_i|_t|_d|_|jjrt^j|j"t^j{|jd|_d|jX|_t|j|_|j9rt|j|j5nd|_d|_t|_d|_d|_t^n|_t^j|j$dft^j}d|jd|_d|_d|_d|_d|_d|_d|_|jXrt^n|_t^j_`|_t^j|j$|jXft^j}d|jd|_|j>rt^n|_t^j_`|_t^j|j$t^j}d|jd|_d|_d|_i|_d|_dS)Nr)set_cpu_offload_max_bytes@poolingFrexternal_launcher) logprobs_modengramr)rrrunnersuffixeagle3medusa)rrz%Unknown speculative decoding method: r) max_num_reqs max_model_lenmax_num_batched_tokensr pin_memoryr block_sizeskernel_block_sizesis_spec_decode logitsprocs!logitsprocs_need_output_token_idsrAcp_kv_cache_interleave_sizedtype)rnumpyrrr)rrr)r model_config cache_configcompilation_config lora_config load_configparallel_configscheduler_configspeculative_configobservability_config vllm.model_executor.models.utilsr rcpu_offload_gbrrWrrrY cache_dtypekv_cache_dtype runner_typerAenable_prompt_embeds"is_multimodal_raw_input_only_modelis_multimodal_pruning_enabledrcalculate_kv_scalesdecode_context_parallel_sizedcp_world_sizer rank_in_groupdcp_rankrmax_num_tokens max_num_seqsrdistributed_executor_backendlenr!ranksbroadcast_pp_outputget_num_attention_headsnum_query_headsget_inputs_embeds_sizeinputs_embeds_sizeattention_chunk_size uses_alibi use_alibidisable_cascade_attncascade_attn_enabledis_mm_prefix_lmrC mm_registry uses_mropeuses_xdrope_dimsupports_multimodal_inputssupports_mm_inputsis_encoder_decodermax_num_encoder_input_tokensmax_encoder_lenasync_schedulinguse_async_schedulingrr sampler eplb_state kv_cachescross_layers_kv_cachecross_layers_attn_backend attn_groups encoder_cacheuse_aux_hidden_state_outputs is_last_rankmethod"vllm.v1.spec_decode.ngram_proposerrdrafteruses_draft_modelrr use_eaglereagle3_use_aux_hidden_stater ValueErrorrrejection_samplernum_spec_tokensnum_speculative_tokensdraft_model_configeffective_drafter_max_model_lenrequestsnum_prompt_logprobsrrr comm_streamlogits_processorstuplermaxget_vocab_size block_sizerrr input_batchrprepare_inputs_eventrcudagraph_capture_sizescudagraph_moderNONEsortedcudagraph_batch_sizes_init_device_propertiesencoder_timing_registry threadingLock_encoder_timing_lock _make_bufferint32 input_idsint64 positionsquery_start_locseq_lensencoder_seq_lensdcp_local_seq_lens inputs_embeds is_token_idsdiscard_request_masknum_decode_draft_tokensnum_accepted_tokensis_mm_embed_buffersis_mm_embed_idxmrope_positionsxdrope_positionsintermediate_tensorsnparange arange_npshared_kv_cache_layersset'kv_sharing_fast_prefill_eligible_layers&kv_sharing_fast_prefill_logits_indiceskv_sharing_fast_prefillzerosuniform_decode_query_lenrfcudagraph_dispatcherrD mm_budgetreorder_batch_thresholdrunner_only_attn_layers_draft_token_ids_draft_token_req_idstransfer_eventemptysampled_token_ids_pinned_cpuvalid_sampled_token_count_event%valid_sampled_token_count_copy_streamdraft_token_ids_eventdraft_token_ids_copy_streamvalid_sampled_token_count_cpudraft_token_ids_cpuexecute_model_statekv_connector_outputmamba_state_idxlayerwise_nvtx_hooks_registered) rrrr r"r#r(r'r draft_configrjcustom_logitsprocsrrrrLs                                              zGPUModelRunner.__init__rrNcCs:||_|jr|jj}|dus|jdur|j|_dSdSdSr)rr)rerf)rrrrrrupdate_max_model_lens z#GPUModelRunner.update_max_model_lencCs|jr |jdSdS)z| Clear the multi-modal cache that was used during profiling, but no longer needed during inference. N)r reset_cacherrrrreset_mm_cacheszGPUModelRunner.reset_mm_cachecCs|jdS)zClear the GPU-side encoder cache storing vision embeddings. This should be called when model weights are updated to ensure stale embeddings computed with old weights are not reused. N)rXrrrrrreset_encoder_cachesz"GPUModelRunner.reset_encoder_cachec Cs|jjds dSt|dg}|D] }|dur|qd}d}|jj}|D]C\}}t|t t frkd\}} |D]} t || rPt|| } t| t j rP| |q9|D]} t || rjt|| } t| t j rj| | qSq(dS)a Re-initialize the KV cache and FP8 scales after waking from sleep. 1. Zero out the KV cache tensors to remove garbage data from re-allocation. 2. Reset Attention layer scaling factors (_k_scale, _v_scale) to 1.0. If these are left at 0.0 (default after wake_up), all KV cache values become effectively zero, causing gibberish output. fp8NrT)_k_scalek_scale)_v_scalev_scale)?r)r#r- startswithgetattrzero_r$static_forward_contextitems isinstancer+r,hasattrrrfill_) rrT cache_tensor k_attr_names v_attr_names attn_layersnamemodule k_scale_val v_scale_valattrparamrrrinit_fp8_kv_scaless4          z!GPUModelRunner.init_fp8_kv_scales num_tokenscCst|tr-|jr|jjddd|fS|jdkr%|jjddd|fS|jjd|S|jr:|jjdd|fS|jdkrI|jjdd|fS|jj|SNr)rrrIrgpurJrr)rrrrr_get_positions s    zGPUModelRunner._get_positionsT)rsizerrcGst|||j|j|dS)N)rrr with_numpy)rrr)rrrrrrrr{szGPUModelRunner._make_bufferc Cstttf}|js |S|jj}|j}tttf}t|D]\}}|j dur7|j d}dur7|||<q t |dkr@|S|j j d|}g} t|D]}| |||} t||| k} | | qNt| j|jd|d<|S)Ncompressed_token_type_idsrrtoken_type_ids)rrr rAronum_reqsget_pooling_paramsr enumerate extra_kwargsgetr;rrrangerrappendconcatrr) r model_kwargsrpooling_paramstoken_type_id_requestsrr token_typesrrposidsrrr_init_model_kwargs%s2       z!GPUModelRunner._init_model_kwargsrrcCs8t|jjdkr dS|jdurt|j||jddSdS)a[ Update the order of requests in the batch based on the attention backend's needs. For example, some attention backends (namely MLA) may want to separate requests based on if the attention computation will be compute-bound or memory-bound. Args: scheduler_output: The scheduler output. rN)decode_threshold)r;kv_cache_configkv_cache_groupsrrdro)rrrrr_may_reorder_batchGs  z!GPUModelRunner._may_reorder_batchcCstj|j|_|jj|_dS)z;Initialize attributes from torch.cuda.get_device_propertiesN)rrget_device_propertiesrdevice_propertiesmulti_processor_countnum_smsrrrrrvasz&GPUModelRunner._init_device_propertiescCstjdSr)rrrrrrr _sync_devicegzGPUModelRunner._sync_devicec'Cs|jD]}|j|d|j|dq|jD]}|j|q|jD] }|j|dq#|j }|jj }|j j }|||}|D]}|j|qDg}|j D]} | j}||jvrh||| } || qR| j} | j} | r| jtjkrtj|jd} | | jnd} |jr| dusJ| j}|dusJdtt|}|j !|}|"| t#|| j$| j%| j&| | | | j'| j(g| j)d } | |j|<| r| j*dur| j*dkr|jj+n| j*|j|<|j,r|-| |j.dkr|/| || qRt0j1}|j }|j2}|3}t4|j5D]:\}}|j|} |j(|}|j6|}||j v}|j7|}|jj 8|}| j9rd|j:rd|dur=d| _9n'|jj;dusFJ|jj;|}||d}| j9|}||8}| j<=dg||| _(|s|j>sqg}nN|j>|}|t?|| j@}|dkr| j<|dn2|dkr| j<=|| dn!|t?| j<kr| j<|d=|dur|jjA||} | |jjB|<|s|durtC| j'|D] \}!}"|!=|"qn|dusJ|dusJ|| _'|dur |j:r|dkr|jD|}#|#| d| _<|| q||jjE|<|dur|jjFG|||s:|}$|t?|}%||jjH||$|%f<|%|jjB|<|jI| |q|D]}&|jJ|&|jI|&|qE|jK|L||jMdS)abUpdate the cached states and the persistent batch with the scheduler output. The updated states are used by the `_prepare_inputs` function to create the input GPU tensors for the model. The SamplingMetadata is updated and copied to the GPU if there is a new/resumed/paused/finished request in the batch. Nrz!You did not set `task` in the API) req_idprompt_token_ids prompt_embeds mm_featuressampling_paramsr generator block_idsnum_computed_tokensoutput_token_ids lora_requestrrr)Nfinished_req_idsrgpoprhroremove_requestfree_encoder_mm_hashesrXnum_scheduled_tokenskeysreq_id_to_indexscheduled_cached_reqsresumed_req_idsscheduled_new_reqsr_update_streaming_requestrrr sampling_typerJ RANDOM_SEEDr Generatorr manual_seedseedrAtaskrr@ get_modelpoolerget_pooling_updatesapplyrrrrrrrprompt_logprobsrrI_init_mrope_positionsrJ_init_xdrope_positionsr!rZscheduled_spec_decode_tokens_get_valid_sampled_token_countrreq_ids new_block_idsnum_output_tokensrprev_num_draft_lenrQprev_req_id_to_indexrextend new_token_idsr;rnum_prompt_tokensnum_tokens_no_specr all_token_idsnum_computed_tokens_cpu block_table append_row token_ids_cpuupdate_req_spec_token_ids add_requestcondenserrefresh_metadata)'rrrmm_hashscheduled_req_idscached_req_idsrunscheduled_req_ids reqs_to_add new_req_data req_staterrrrmodel to_updaterZreq_datascheduled_spec_tokensvalid_sampled_token_countrrrresumed_from_preemptionr req_indexprev_req_index num_accepted num_rejectedrnum_new_tokensend_idxrnew_idsresumed_token_idsstart_token_indexend_token_indexrequestrrr_update_statesjs                                                  zGPUModelRunner._update_statesrc Cs|jr|jjs dStj|tj|ddfd|jdgdddk d }t |D] \}}||j j|<q-|jjdkrTt||j|j |j|j|jj|jdSdS)aUpdate the cached states after model execution. This is used for MTP/EAGLE for hybrid models, as in linear attention, only the last token's state is kept. In MTP/EAGLE, for draft tokens the state are kept util we decide how many tokens are accepted for each sequence, and a shifting is done during the next iteration based on the number of accepted tokens. Nrrrrdimalign)r)r" is_hybridrcatfullrrrargmaxrrrronum_accepted_tokens_cpur#mamba_cache_moderpostprocess_mambarrgrr$rr)get_mamba_state_copy_func)rrrrrrrrr"_update_states_after_model_executeas@     z1GPUModelRunner._update_states_after_model_executerr'cCs||j||j|}|j|_|j|_|j|_|j|_|j|_|j|_|j |_ t |j|j|_ |j |jr<|||S)ajUpdates streaming session request from `scheduled_new_reqs`. Removes the request from InputBatch (if present), updates the cached state, and prepares it for re-addition to the batch. NOTE: prompt_token_ids includes intermediate output tokens - tokens previously generated but now are input context (part of the prompt). )rorrgrrrrrrrrPrrrrIr )rrr'r(rrrrs   z(GPUModelRunner._update_streaming_requestr(cCsP|}t|s Jd|jdusJdtt|}||j|j\|_|_dS)Nz"M-RoPE support is not implemented.z1M-RoPE requires prompt_token_ids to be available.) rr;rrr6get_mrope_input_positionsrrmrope_position_delta)rr(r) mrope_modelrrrr s  z$GPUModelRunner._init_mrope_positionscCsJ|}tt|}|jdusJdt|sJd||j|j|_dS)Nz2XD-RoPE requires prompt_token_ids to be available.z#XD-RoPE support is not implemented.)rrr8rr?get_xdrope_input_positionsrr)rr(r) xdrope_modelrrrr s   z%GPUModelRunner._init_xdrope_positionscCs|r|jsiSttttf}|jD]}|jD]}|jdur(||j |jfqqi}t ||j |j dD] \}}}| |q5|S)Nrr)r1rrkrrFrrdatarmodalityrHrrupdate)rr mm_kwargsreqfeaturemm_kwargs_combined_mm_kwargs_grouprrr_extract_mm_kwargss"     z!GPUModelRunner._extract_mm_kwargsnum_seqscCs:|jsiS|j}|dusJ|jsiS|}|||Sr)r1rmm_max_toks_per_itemget_modality_with_max_tokens_get_mm_dummy_batch)rrWrdummy_modalityrrr_dummy_mm_kwargss  zGPUModelRunner._dummy_mm_kwargs cumsum_dtypecCs@tj||d}|d}t|||}|jd||}||fS)zGet the cumulative sum and batched arange of the given array. # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) # Equivalent to but faster than: # np.concatenate([np.arange(n) for n in num_tokens]) rrN)rcumsumrepeatr)rrr] cu_num_tokenstotal_num_tokenscumsums_offsetsrrrr_get_cumsum_and_aranges  z%GPUModelRunner._get_cumsum_and_arangetotal_num_scheduled_tokensr`cCs|jjdur|j||jr|j||j|dS|jj}|dus'Jg}g}g}g}d} d} d} |j} |jj D]U\} }| | }dur| |t | | d}| |7} ||d}| |||t||d|d||j}|t|||| ||kM} t| |} q>t |}|| }||kr|j||jr|j||j||dkrdS| r| |dkr|jjd|j|jjd|dfdd|jrd|jjd|<dStj|tj|jdj|jdd}tj|tj|jdj|jdd}|jjjd||jj|dfd |jdus|s dSt|jtjs*Jtj|tj|jdj|jdd}tj|tj|jdj|jdd}|jjtjd }|jjjd|| |d dS) aPrepare the input IDs for the current batch. Carefully handles the `prev_sampled_token_ids` which can be cached from the previous engine iteration, in which case those tokens on the GPU need to be copied into the corresponding slots into input_ids.NTrrrrr)rr)r<indexsrcr)!roprev_sampled_token_idsr} copy_to_gpur0rrrrrrrrr;itemrrrcrlrcopy_rtensorr~rrrscatter_rrrr|flatten)rrrdr`rsample_flattened_indicesspec_flattened_indicesprev_common_req_indicesprev_draft_token_indices indices_matchmax_flattened_indextotal_num_spec_tokensr,r cur_index prev_index draft_lenflattened_indexstartnum_commmon_tokenstotal_without_specsampled_tokens_index_tensorprev_common_req_indices_tensordraft_tokens_index_tensorprev_draft_token_indices_tensordraft_token_idsrrr_prepare_input_idss                  z!GPUModelRunner._prepare_input_idsFr kv_cache_specrfor_cudagraph_capturec Cst|tsdSd|jjd|<|D])}|jj|}|j|}|jdur*d|jj|<qtdd|jD}||jj|<q|rNt |j j d|j } | |jjd|<|j ||jjd|} |jjd|} | | fS)NNNrcs|]}|jjVqdSr) mm_positionlength)rrRrrr  z7GPUModelRunner._get_encoder_seq_lens..max_source_positions)rrirrrorrgrsumrr" hf_configrOrhr) rrrrrrr/r(encoder_input_tokensrOrencoder_seq_lens_cpurrr_get_encoder_seq_lenss0      z$GPUModelRunner._get_encoder_seq_lenscs|j}|dks Jjj}|dksJjj|tjd||}|\}}j jd|}tj jj |||dj rH |jdkrR|||jjjd} t| } tjjjd| jjd|djrjj} tj| d| jjd|djjrd} t|D][} || }| jjvr| |7} q|dkr| |7} qjj| }jj | }||jdkr| |7} q||}t||jd}||}|dkrjj| | | |||| |7} qjj!||jj"|dj#jd<|j#jd|d<j#j|dd$|dj#%j#j&d|d}jj d||j'jd|<j'j|d$dj'%fddjj(D}tj)|tj*d}j'jd||kj+jd|<j+%|,|||j rj-j&ddd|fj j-jddd|fd d n(jdkrˆj.j&ddd|fj j.jddd|fd d nj %|t/|j0dk}|s|ddd}d}tj1|tj*d}n`tj2|tj*d}tj3|dtj*d}|j04D]$\}}jj5|} t/||| <jj | jj6| krt/||| <q7||}|j8}|d}|j9jd|<j9j|d$dj9%j:r^t;|jksVJ?j||||fS) z] :return: tuple[ logits_indices, spec_decode_metadata, ] rN)rrrcsg|]}j|jqSr)rgr)rrrrrr2z2GPUModelRunner._prepare_inputs..rTr)@rdrorrcommit_block_tablerr_rrcraddrrI_calc_mrope_positionsrJ_calc_xdrope_positionsrrr from_numpy index_selecttoken_ids_cpu_tensorrmr}rr0is_token_ids_tensorrreq_prompt_embedsrminrrjcompute_slot_mappingcommit_slot_mappingrfillrhrrrarrayr|rrrrr;ronesrr@rrr_calc_spec_decode_metadatalogits_indicesrr%rrr(rset_active_loras)rrrrdr req_indicesr`r positions_np token_indicestoken_indices_tensorr output_idxreq_idx num_sched req_embeds start_posend_pos actual_endactual_num_schedrr num_tokens_npuse_spec_decoderrnum_sampled_tokensnum_draft_tokensrrrrrr_prepare_inputss                             zGPUModelRunner._prepare_inputs max_query_lennum_tokens_paddednum_reqs_padded ubatch_slicesrrcascade_attn_prefix_lensrc $s@tjjdkr idfSp|pdurdusJi|dur/ddtt|Dr5j} n jjd} raj j dj jd<j jd dj jjdtffdd }| dusxJ|d}| d}jjr|d|_tjjddjjddjjdjjdj jd|| ||d d }jdkrtjjdjjjjjjd<jjddj jjd|_jjd|_|durj j!r|"d|_#$||_%i ddtd td td tdBddf  fdd }d}t&D]v\}}t'|}j(| pIi|j)d\|_*|_+|dkrc|||_,| ||_j-r|durt.j/t0rj/j1d|j2vr|}n|}ttj3|D]$}|durt&t4||D] \}}|||||qq||||qq;j5ri}j j6D]'}g}j7|}|j8D]}|j9}|:} |;| qȈj j<|}!|||!<qt.t=rD]}"|">D]}#||#_?qqn >D]}#||#_?q|durks|kr|@|}|fS)zQ :return: tuple[attn_metadata, spec_decode_common_attn_metadata] rNcSg|]}tqSr)rrrTrrrrz.r kv_cache_gidcsndurdus J|j}t|tr!tjdftjjd}n jj|}| }| d|S)Nrr!r) rrrjrrr|rrorget_device_tensorr)rrblk_table_tensor blk_table)rrrrrrr_get_block_tables    zBGPUModelRunner._build_attention_metadata.._get_block_tableT) rquery_start_loc_cpur _seq_lens_cpu_num_computed_tokens_cpurnum_actual_tokensr max_seq_lenblock_table_tensor slot_mappingcausalattn_gidcommon_attn_metadataubidrc sJj||}||p d}|j}t|tr |j|jd}|t|f}r.||nd}i} rSt|trS|dusAJdt j j dj j dd} r[||} n%|vrn|jrn|||j|j} n|jd||d| } |jr| |<|durtt sJ} n ttsJ|} |jD]} | | | <qdS)Nrz$UBatching not supported with GDN yet)rnum_decode_draft_tokens_cpu)common_prefix_lenrr)rWget_metadata_builderrrrqkv_cache_specs layer_namestyperarrrrrbuild_for_cudagraph_capturesupports_update_block_tableupdate_block_tablerrbuildr) rrrr attn_groupbuilderr cache_keycascade_attn_prefix_lenextra_attn_metadata_argsattn_metadata_iattn_metadata_dict layer_name) attn_metadatacached_attn_metadatarrrrrrrr_build_attn_group_metadatas^     zLGPUModelRunner._build_attention_metadata.._build_attn_group_metadata)rr)Ar;rrrrrrrlrirorBrrrhrr"enable_return_routed_expertsrrrr_rrnum_computed_tokens_cpu_tensorr5rcr7r'rrrdcp_local_seq_lens_cpur#rrnum_logits_indices _prepare_kv_sharing_fast_prefilllogits_indices_paddedrrrrrrrr)rr]rattn_layer_namesrrWrrGrrgrrextract_embeds_rangerrrvaluesmm_prefix_rangeunpadded)$rrrrrrrrrrrrrrrblock_table_gid_0slot_mapping_gid_0cm_baserrrkv_cache_groupcmrr_cmreq_doc_rangesrimage_doc_rangesr( mm_featurepos_info img_doc_ranger ub_metadata _metadatar) rrrrrrrrrrr_build_attention_metadatas      ?                z(GPUModelRunner._build_attention_metadatarnum_common_prefix_blocksc Csd}t|jj}ddt|D}t|D].}|j|D]&}t|jtr'd} n||||||j| } || | || dkO}qq|rH|SdS)z :return: Optional[cascade_attn_prefix_lens] cascade_attn_prefix_lens is 2D: ``[kv_cache_group_id][attn_group_idx]``, None if we should not use cascade attention FcSg|]}gqSrrrrrrrszDGPUModelRunner._compute_cascade_attn_prefix_lens..rN) r;rrrrWrrrj _compute_cascade_attn_prefix_lenrr) rrrruse_cascade_attnnum_kv_cache_groupsrrrrrrr!_compute_cascade_attn_prefix_lens}s(    z0GPUModelRunner._compute_cascade_attn_prefix_lensattn_metadata_builderc Cs||j}|dkr dSt||}||j|j}t|tp(t|to(|jdu}t|tp7t|to7|jdu}t|ts?J|j |||j |j |j |||j |jd } | rV|SdS)a6Compute the length of the common prefix for cascade attention. NOTE(woosuk): The common prefix length returned by this function represents the length used specifically for cascade attention, not the actual number of tokens shared between requests. When cascade attention is disabled (use_cascade=False), this function returns 0 even if requests share common tokens. Additionally, the common prefix length is truncated to a multiple of the block size and may be further truncated due to implementation details explained below. Args: num_scheduled_tokens: Number of tokens scheduled per request. num_common_prefix_blocks: Number of shared KV cache blocks. Returns: int: Length of common prefix in tokens. rN) r query_lensr? num_kv_headsrDuse_sliding_windowuse_local_attentionrr5)rnrrrprksliding_windowrhrBrguse_cascade_attentionr?rrDrr5) rrrrrrrrr use_cascaderrrrs6 )     z/GPUModelRunner._compute_cascade_attn_prefix_lencCs6d}t|jjD]\}}|j|}|jdusJ|jj|}|j|}t|j|j }|||kr?t d||} t d|| } n|} d} || | ksKJ| dkrt|} || } |} || }|jdd| |f|jj dd| | f<|| 7}| dkr|} || } |j dusJt j|jj| |j || | d|| 7}qdS)Nr)r out_offsetrH context_lenr3)rrorrgrrrrPrrrlrrHr/get_next_input_positions_tensorr)rr mrope_pos_ptrrerrQrrrprompt_part_lencompletion_part_len dst_startdst_end src_startsrc_endrrrrsL    z$GPUModelRunner._calc_mrope_positionscCs$d}t|jjD]\}}|j|}|jdusJ|jj|}|j|}t|j|j }|||kr?t d||} t d|| } n|} d} || | ksKJ| dkrt|} || } |} || }|jdd| |f|jj dd| | f<|| 7}| dkr|} || } t j |jj| || | d|| 7}qdS)Nr)rr r r3)rrorrgrrrrPrrrlrr0r r)rrxdrope_pos_ptrrerrQrrrr rrrrrrrrr3sH    z%GPUModelRunner._calc_xdrope_positionsrcu_num_scheduled_tokensc Cs|d}|j|tjd\}}t|||}||7}|d}|j|tjd\}}t|||} | |7} t|j|jdd}t|j|jdd}t|j|jdd}t| j|jdd} t|j|jdd}|jj |} | | d} t | | ||| ||dS)Nr)r]Tr)rrcu_num_draft_tokenscu_num_sampled_tokenstarget_logits_indicesbonus_logits_indicesr) rcrr|r_rrrrr}rrr) rrrrrrrrrrrrrrrbsT         z)GPUModelRunner._calc_spec_decode_metadatacCs|jdusJ|jd}|dksJ|jd|||j|d|d|jj|dd\}}|j}|jd|}|S)NrrT disable_full)rrrjrrirdispatchr)rr num_logitsrT batch_descnum_logits_paddedrrrrrs    z/GPUModelRunner._prepare_kv_sharing_fast_prefillc Cs|j}|s gggfStt}ttttf}ttttf}|D]0\}}|j|}|D]$} |j| } | j dur=q0| | j | | j | j f| || j fq0q%|||fS)aBatch multimodal inputs from scheduled encoder inputs. Args: scheduler_output: The scheduler output containing scheduled encoder inputs. Returns: A tuple of (mm_hashes, mm_kwargs, mm_lora_refs) where: - mm_hashes: List of multimodal hashes for each item - mm_kwargs: List of multimodal kwargs for each item - mm_lora_refs: List of (req_id, placeholder_range) for each item N)scheduled_encoder_inputsrrrkrFrGrrgrrMr identifierrNr) rrr mm_hashesrP mm_lora_refsrencoder_input_idsr( mm_input_idrrrr_batch_mm_inputs_from_schedulers"       z.GPUModelRunner._batch_mm_inputs_from_schedulerc# s|\}}}|s gStjojjo|j}ttj}jrj rg}g}t } g} |D]@\} } j j | } tj j| }j| }||||g|| ||dkrqj j|}|durq| |q1tt|t|dtjd}j | |tjdrfdd| D}ttj|tj dtj|tj d}tt|!t|dtj"d}j | |g}d}t#|j$j%d D]\}}}j&r*|d kr*|d kr*t't(j)}t*|D]?}|||}+||||d $t,t#|gj$j%d \}}}|j-di|}||Wdn 1s!wYq|} n!+|||||j-di|} Wdn 1sFwYt.| |d || ||7}qt/||D]\}!}"|"j0|!<t12d |!3j0|!q`|S)NrT) is_prefillrget_num_mm_connector_tokenscsg|]}j|qSr)r)r')rrrrrr1 s z6GPUModelRunner._execute_mm_encoder..r) index_mappingprompt_mappingr&rrLvideorexpected_num_itemszFinish execute for mm hash %sr)4r%rr*enable_mm_processor_statsrrr7r)r% lora_managersupports_tower_connector_lorarrorrrequest_lora_mappingget_num_mm_encoder_tokensget_num_embedsrrlora_id_to_lora_requestrrr)rkr*TOWERset_active_adaptersrrr_rr|r CONNECTORrHrrr2rrrrtimed_encoder_operationnextembed_multimodalrrrXloggerdebugmaybe_save_ec_to_connector)#rrr!rPr" should_timer)prompt_lora_mappingtoken_lora_mapping lora_requestsencoder_token_countsrrrlora_idrr tower_mappingpost_op_countsconnector_token_mappingconnector_mappingencoder_outputscurrent_item_idxrN num_itemsrUcurr_group_outputs_lst video_idxvideo_mm_kwargs_itemrTmicro_batch_mm_inputsmicro_batch_outputscurr_group_outputsr"rrrr_execute_mm_encoders                       z"GPUModelRunner._execute_mm_encoderrshift_computed_tokenscCsV|j}d|j|_|j|j}ttj}|j}d|d|<d}d}d} |jjD]} g} |j | } |j | } | j |}| j D]}|j }|j}|j}||| krQn|||krXq>t||d}t||| |}||ksnJ|||\}}||kr{q>|j}|j|d}|dusJd|d|j}dur|||}|||}n|||}|||}|durd|||||<n||||||O<| |q>|jr|jr| jdusJd}|jj| j| | j| j d\} }}| j||| _| | || 7}q(|!|}|r|"||j!|| r'|#||j$!|||fS)NrFrzEncoder cache miss for .T)r}multimodal_embeddingsrr)%rdrrrrrrrorrrgrrroffsetrrlrget_embeds_indices_in_ranger rXris_embedrr2rIrr)recompute_mrope_positionsrrjrHrrhrrr)rrrQrdis_mm_embed_buf mm_embeds is_mm_embed req_start_idxshould_sync_mrope_positionsshould_sync_xdrope_positionsr mm_embeds_reqrr(rrrrnum_encoder_tokens start_idxr4curr_embeds_startcurr_embeds_endr"encoder_outputrVmm_embeds_item req_start_posnew_mrope_positions new_deltarrr_gather_mm_embeddings s                           z$GPUModelRunner._gather_mm_embeddingscCs2t|ds tdt|jttfr|jS|jS)Nr)z2Cannot get model before model has been initialized)rrarr)rrunwraprrrrr s  zGPUModelRunner.get_modelcCsX|}tt}t|r|dt|r!|jrdgS|dt|r*|d|S)Ngenerate transcriptionrealtime)rrrLrBrr>supports_transcription_onlyr=)rr)supported_tasksrrrget_supported_generation_tasks s    z-GPUModelRunner.get_supported_generation_taskscCsZ|}t|s gSt|j}d|vr+t|jjdd}|dkr+|dt d|S)Nscore num_labelsrrz.Score API is only enabled for num_labels == 1.) rrArrget_supported_tasksrr"rremover: debug_once)rr)rnrqrrrget_supported_pooling_tasks s  z*GPUModelRunner.get_supported_pooling_tasks.cCsFtt}|jjdkr|||jjdkr||t|S)Nrjr )rrNr"r/rrorurk)rtasksrrrrr' s   z"GPUModelRunner.get_supported_tasksr sync_selfcs|jdusJ|jjjt|j|rC|dusJ|D]$\}}|dko'}|r.n}|j|d|j|d|ddqtfdd|jDS)NresidualTrcs:i|]\}}||dkrr|dn|dqS)rxNrrkvis_rsrtprr H s  zFGPUModelRunner.sync_and_slice_intermediate_tensors..)rrr'tensor_parallel_sizerrrjrK)rrrrwrzr{ is_scatteredcopy_lenrr|r#sync_and_slice_intermediate_tensors1 s      z2GPUModelRunner.sync_and_slice_intermediate_tensorsis_dummy is_profilecCsJ|jjsdS|jdus J|}t|sJ|jj|||jjjddS)zN Step for the EPLB (Expert Parallelism Load Balancing) state. N) log_stats)r' enable_eplbrSrr9step eplb_configlog_balancedness)rrrr)rrr eplb_stepP s  zGPUModelRunner.eplb_steprnum_scheduled_tokens_nprc Cs|jj}|t|jjksJd|d|}|jjd|}|j}|j|||jdt t |j }|j ||d} ddt ||jD} t|jj|jj|d} | dus[t| scdg|| _| S|jrot| | | |jdStdd | } d dt | | D| _|| S) NzEEither all or none of the requests in a batch must be pooling requestrrpooling_metadatacSsg|]\}}||kqSrr)rseq_len prompt_lenrrrry sz(GPUModelRunner._pool..)rrr)rrrrcSrrrrrrrr rz&GPUModelRunner._pool..cSrrrrrrrr r)rorr;rrrget_pooling_metadatabuild_pooling_cursorrrr@r)rr prompt_lensryrrranyrrQrrrQr) rrrrrr seq_lens_cpurr)rrrrrr_pool` sR       zGPUModelRunner._poolcCs*|jjj}|jjjr|dkrt||S|SNr)rr'rr$ pass_config enable_sprS)rrtp_sizerrr_pad_for_sequence_parallelism s  z,GPUModelRunner._pad_for_sequence_parallelismcCs6|jjr |jjd|}nd}|jjd|}||fSr)r)requires_raw_input_tokensr}rr)rrr}rrrr_prepare_mm_inputs s z!GPUModelRunner._prepare_mm_inputsnum_input_tokenscCs|j}tj}|jj}d}|jre|re|se|j||jd}||| |\}} Wdn1s3wY|j j |j j d||| d} |jj d|| ||\} } i|||} nJ|jr|r|jj d|jddd}|dkr|j j |}|j j |d}||jj |<|jj d|} |} d} n|j j d|} d} |} |jr|jj ddd|f}n|jdkr|jj ddd|f}n|jj d|}|rd}n |dusJ|||d}|r|jr||}| d |i| | ||| |fS) NrX)rS is_multimodalF)as_tuplerr)r}TrG) rdr! is_first_rankr"rMrLmaybe_get_ec_connector_outputrXrPrhr)embed_input_idsr}rrrjrrrVr0rnonzerosqueezenumelrIrrJrrrrrO)rrrrrrrMrrYrZinputs_embeds_scheduledr}rr token_ids_idx token_idstokens_to_embedsrrGrrr _preprocess s|           zGPUModelRunner._preprocessrrcCsd|jj}|j|dur|j||dS|jr(|jdur(|\}}|j|||d||}|S)Nrsampling_metadata) rorupdate_async_output_token_idsrRrQr_get_draft_token_ids_cpuupdate_async_spec_token_idsrb)rrrrrrTsampler_outputrrr_sample# s"   zGPUModelRunner._samplercsbi}tjr ||}|jj}t|jjd|d} | D]} |jj t | } | dur5| | dq|jj } |jj} |jjd}|j}|j}g}d}|js|jd}|dkry||}| D] } |t | qe|durx|}n:tj||jj| |d\}}n,g}| }t||jjdur|jddksJ||j_fddt|jj D|j_|jj }t|D]a}|jr|vrdgnd}n||}|rt |nd}|sq|jj!|}||}||j"ksJd|d |j"||jj#|||f<d |jj$|||f<||jj!|<||}|j%|}|j&'|q|(|d||j)}||||| | |fS) Nrrrrcsi|] \}}|vr||qSrr)rrrinvalid_req_indices_setrrr s z4GPUModelRunner._bookkeeping_sync..zGSampled token IDs exceed the max model length. Total number of tokens: z > max_model_len: T)*envsVLLM_COMPUTE_NANS_IN_LOGITS_get_nans_in_logitsrorrrr generatorsrr set_offset get_offsetrrrrrrrQ_to_listrrrrrrrrgrrrr;rrrrrgrr_get_prompt_logprobs_dictr)rrrrrrrnum_nans_in_logitsr"discard_sampled_tokens_req_indicesrgenreq_ids_output_copyreq_id_to_index_output_copyrrrrrrrrr sampled_idsnum_sampled_idsr`r4rr(prompt_logprobs_dictrrr_bookkeeping_syncA s                 z GPUModelRunner._bookkeeping_syncccsD|jdur dVdS|jz dVW|jdS|jwr)rprrrrrrsynchronize_input_prep s  z%GPUModelRunner.synchronize_input_prepr}rrrcKs|jd||||d|S)aMHelper method to call the model forward pass. This method can be overridden by subclasses for model execution. Motivation: We can inspect only this method versus the whole execute_model, which has additional logic. Args: input_ids: Input token IDs positions: Token positions intermediate_tensors: Tensors from previous pipeline stages inputs_embeds: Input embeddings (alternative to input_ids) **model_kwargs: Additional model arguments Returns: Model output tensor r}rrrNr)r))rr}rrrrrrr_model_forward szGPUModelRunner._model_forwardmax_num_scheduled_tokensrforce_uniform_decodecCs |dur||ko |||kS|S)zn Checks if it's a decode batch with same amount scheduled tokens across all requests. Nrrrrrrrrr_is_uniform_decode s  z!GPUModelRunner._is_uniform_decoderallow_microbatching force_eagerforce_has_loraforce_num_active_lorasnum_encoder_reqsc  stj|j|||djjo| dk} | dur| ntjj| dur'dkn| |fdd} | |p>| \}}|jj j j rX|jj j jdksXJdd\}}j j jdkrj jtjk}t|j ||||jd\}}}|durj j}t||| |tjjkd \}}|jksJd}j jjrt||j|j|t|d }|||||fS) Nrrcs(s jj||dStjtfS)N)rhas_lorauniform_decodernum_active_loras)rrrrsr&)rrrrrrrrrrr1 szGGPUModelRunner._determine_batch_execution_and_padding..zQSequence parallelism requires num_tokens to be a multiple of tensor parallel size)FNr)num_tokens_unpaddedr'rallow_dp_paddingrr num_scheduled_tokens_per_requestrrr)num_unpadded_tokensnum_padded_tokens num_paddings runtime_mode)rrr"rMr;ror3rrr$rrrr'rdata_parallel_sizerrrrsrvaluedata_parallel_rankrri PIECEWISEr*cudagraph_metricsrr)rrrrrrrrrrrrhas_encoder_outputdispatch_cudagraphrrbatch_descriptor should_ubatchnum_tokens_across_dprsynced_cudagraph_modedp_rankrrrr&_determine_batch_execution_and_padding s          z5GPUModelRunner._determine_batch_execution_and_paddingcCsr|jjjr5|js7|jjtjkrt d|jjj t j kr#t ddSt }||j|jjjd|_dSdSdS)z Register layerwise NVTX hooks if --enable-layerwise-nvtx-tracing is enabled to trace detailed information of each layer or module in the model. zlayerwise NVTX tracing is not supported when CUDA graph is turned off; you may observe part or all of the model missing NVTX markerszylayerwise NVTX tracing is not supported when CompilationMode is STOCK_TORCH_COMPILE, skipping function hooks registrationTN)rr*enable_layerwise_nvtx_tracingrr$rrrrsr:rtmoderSTOCK_TORCH_COMPILErVregister_hooksr) __class__r)r pyt_hooksrrr_register_layerwise_nvtx_hooks s$  z-GPUModelRunner._register_layerwise_nvtx_hooksrzUBatchSlices | NonecstdrjdurtjjdksdSdtffdd fdd tjjD}i}tjjD]\}}||} |jD]} | || <q@q5|durng} |D]} i} |D] \} } | | j| | <qX| | qP|| fS||fS) a2 Build slot mappings in both formats needed by the system. Args: num_tokens_padded: Total number of tokens (padded) num_reqs_padded: Total number of requests (padded) num_tokens_unpadded: Actual number of tokens (unpadded) ubatch_slices: Optional ubatch slicing info for DBO Returns: A tuple of: - slot_mappings_by_gid: dict[int, torch.Tensor] for attention metadata - slot_mappings_by_layer: dict[str, torch.Tensor] or list for ForwardContext rNrrrcsvdurdus Jjj|j}t|tr"tjftjjd}nj j |}|j j d}| d|S)Nr!r)rrrrrjrrr~rrorrrr)rrrr)rrrrrr_get_slot_mapping s   z._get_slot_mappingcsi|] \}}||qSrr)rgidrT)rrrr z5GPUModelRunner._get_slot_mappings..) rrr;rrrrr token_slicer)rrrrrslot_mappings_by_gidslot_mappings_by_layerrrrrresultubatchsliced_mappingsr)rrrrrr_get_slot_mappings s0      z!GPUModelRunner._get_slot_mappingsc/s,jdur tdjjjr t}|dur|nt dj r-t r-t j j}tdtrwtjrwjjd}tWdWdWdS1srwY|sjjdkrjjdkrdt stWdWdSjWdWdSjj rj!rJdj"j#}j"j$}fdd |D}t%j&|t%j'd } t(| )} j} *| \} } d}j+rjj,s-| j"j.d|j/}j0| || | |dut1j2d \}}}}}t3d |||||j4}|j#dur(|j#n|}t5|| ||jj6\}}t3d ||t7fddt8j9j:D }|t;j?j9jj@j"jAjBjCjDEt1jFdk}|r{|n|}jG|s|r|n| |s|r|n|| |d\}}jH| |r|nd||r|nd| || |jI||d \}}J||\} }!}"}}#}Wdn 1swYWdn 1swYjKrt;jL}d_Kt1j2}$jjMo|$dk}%tN|j|||||||%d @td+O}&jPd| |"||!d|#}'Wdn 1s,wYWdn 1s.r)rrrrrrzhRunning batch with cudagraph_mode: %s, batch_descriptor: %s, should_ubatch: %s, num_tokens_across_dp: %s+ubatch_slices: %s, ubatch_slices_padded: %sc3s8|]\}}t|jtstddj|DVqdS)csrr)backend forward_includes_kv_cache_update)rgrrrrt s  z9GPUModelRunner.execute_model...N)rrrjallrW)ridspecrrrrs s  z/GPUModelRunner.execute_model..r=rrrrr) rrrrrrrrrrrF)rrcudagraph_runtime_moderrr skip_compiledzgpu_model_runner: forwardrzgpu_model_runner: postprocessrx)all_gather_groupall_gather_tensorsr)rfr)cr RuntimeErrorrr"rr. get_instance clear_bufferr:errorpreempted_req_idsrrhandle_preemptionsrdrrr:rr is_producerrrXrPr|r'r:r _dummy_runrrkv_connector_no_forwardr#rrhrorrrrr|rrlrrF use_ubatchingrrrrr;rr;rr num_ubatchesrrrrrFULLrCrpreprocess_mambarrgr$rr)rErrrrrr3rsrMr'maybe_get_kv_connector_outputrrYr=r!rZrrKrrArcompute_logitsrsend_tensor_dicttensorsr" contiguousbroadcast_tensor_dictr<r)/rrrcapturerrrrrtokensrrrrrrrrrrrrrrrubatch_slices_paddedhas_separate_kv_updatepad_attnrubatch_slices_attnslot_mappings_by_grouprrrr}rrrrhas_encoder_inputr model_outputrrrrr model_output_broadcast_data broadcastedr)rrr execute_model s          !           .            ; zGPUModelRunner.execute_modelgrammar_outputzGrammarOutput | Nonec sj}d_jdur+jrtjdkr|sdS|r"tStt}||_|Sj\ }}}d_|durGt |j |t d |}Wdn1s\wY |jjrt}js|jdkr|jr|jd_d_dj _fdd} j} d} | durduojjjk} | p| o| j } | rtjttBsJ|j}| r| |n@j dur dusJj!|j"j j#j$\}}%||t&j'dj(t&j)d*t+j j,j_j-ddn| } t d .||j/\}}}}}}}Wdn 1s2wY| r>| |t d  0Wdn 1sRwYt d 6j1j2rwt34}|durr|j5j6d nt78d t9||||||j:r|nd||d }Wdn 1swYjs|St dt;||j|j<|j=j j>d}Wdn 1swYt dj ?|j@|jAWd|S1swY|S)Nrzgpu_model_runner: samplec sddusJtd|jj _WddS1s+wYdS)Nzgpu_model_runner: draft)rpropose_draft_token_idsrorr_copy_draft_token_ids_to_cpu)rrrrrrrrrrrr,es   "z=GPUModelRunner.sample_tokens..propose_draft_token_idsFrrT) zeros_onlyzgpu_model_runner: bookkeepzgpu_model_runner: eplbz#gpu_model_runner: ModelRunnerOutput)indicesr) rrrrrrrrrz+gpu_model_runner: AsyncGPUModelRunnerOutput)rrrrrrz-gpu_model_runner: set_async_sampled_token_ids)BrrrQr! world_size1_pp_receive_prev_sampled_token_ids_to_input_batchis_emptyrrrrrorrrFrr=rZ$_pp_broadcast_prev_sampled_token_idsrrrgr)rrcrfr_r^disable_padded_drafter_batchrr]rrrprepare_next_token_ids_paddedrgrr_copy_valid_sampled_token_countrrrr|expandr;rr-rrdrr"rr.r save_captured_expertsrr:rryrLrrrrset_async_sampled_token_idsrr)rr+rrrrrrppr, spec_config propose_drafts_after_bookkeepinginput_fits_in_drafter use_gpu_toksrnext_token_idsvalid_sampled_tokens_countrrrrrrrr async_outputrr.r sample_tokens$s.                      zGPUModelRunner.sample_tokensrcCsLt}|jsJ|dkr|jddksJdtjj||j|jddS)z4Broadcast sampled token ids (GPU) from last PP stagerrz>PP+async expects sampled_token_ids to have shape [num_reqs, 1]rfgroupN) r!rZr<rr distributed broadcastrank device_group)rrr<rrrr5s   z3GPUModelRunner._pp_broadcast_prev_sampled_token_idsc Cst}|jrJ|jj}tj|dftj|jd}tjj ||j |j d||j_ t |jj d|d}t|}i}t|jjD]\}}||vrIq@|||<|j|} dur]| jdq@||j_dS)z6Receive sampled token ids broadcast from last PP stagerr!rFNrr)r!rZrorrrr|rrHrI last_rankrKrgrrrrrrrgrrrr) rr<rrecvdiscard_req_indicesdiscard_req_indices_setrrrr(rrrr3s"   z@GPUModelRunner._pp_receive_prev_sampled_token_ids_to_input_batchcCs&|jr|jsdS|\}}t||Sr)rcrrrt)rrrrrrtake_draft_token_idss   z#GPUModelRunner.take_draft_token_idsr0cCs|jr |js |jjjs dS|jj|_|j}t |sdS|j dus%J|j dus,J|j dus3Jt j}|jd}t j|j *|sZ|j ||j d|j|ddnd|j d|<|j WddS1sqwYdSNrTr)rQhas_structured_output_requestsrorrrrrrr is_tensorrrrrrrrrrjr)rrr0rrrrrrr-s0     "z+GPUModelRunner._copy_draft_token_ids_to_cpucCspt|jtr |j|jjfS|j}|durggfS|jdusJ|jdus&J|j|jdt | |fSr) rrrrorrrrrr;r)rrrrrr5s  z'GPUModelRunner._get_draft_token_ids_cpurArBcCs|jdurdStj}tj|j,|j||}|j}|dus%J|d|jdj |dd|j Wdn1sBwY| d|j _ dS)NrTrr)rrrrrrrrrrjr unsqueezerorg)rrArBrcounts counts_cpurrrr8@s     z.GPUModelRunner._copy_valid_sampled_token_countcCsR|jj}|j}|dus|durgS|j}|dusJ||d|jdSr)rorgrrrrr)rrgsampled_count_eventrVrrrrSs z-GPUModelRunner._get_valid_sampled_token_countrrrrc  s|j|j} | dus J| jdkr6ddlm} t|tsJt|j| s&J|jj||j j |j j | d} | S| jdkrVt|tsBJt|jt sJJ|jj|j || d} | S| jdkrt|tsbJt|jt sjJ|jdt|krv|}n4g} d}|dusJdt|j|D]\}}| |t|d||d7}qtj| |jd } || }|jj||| d } | S| s| rt|jttBsJ| jrt|tsJd |j||j|j |j}n t|tjsJd |j|||j|j |j j!\}}|"||d}|dur8d}|j#j!d}|$}|j%r1|dus"Jtj&fd d|Ddd}n}|d}nv| jrsd}|j'|||j\}|j#j!}|$}|j%rn|dus_Jtj&fdd|Ddd}n@|}n;|j(|||\}}}|j)|j#j!d}|$}|j%r|dusJtj&fdd|Ddd}n|d}|j*r|jj*r|j+|dd}nd}|jj|||||||||| d } | S)Nrrr)rrrz"No spec decode metadata for medusarr)target_hidden_statesrrzGsampled_token_ids should be a python list whenpadded-batch is disabled.zGsampled_token_ids should be a torch.Tensor whenpadded-batch is enabled.cg|]}|dqSrrrhrrrrrz:GPUModelRunner.propose_draft_token_ids..rr;csg|]}|qSrrrZ)rrrrcrYrrrZ)rarrrr)rQ) target_token_idstarget_positionsrXrAtoken_indices_to_samplerrmm_embed_inputsnum_rejected_tokens_gpur),rdr)r[r\rrrr]proposerorrrrrr;rrrrrkrr_r^rrr6prepare_next_token_ids_cpurgrrr7rrr8r}rrYr?prepare_inputsprepare_inputs_paddedrrLrh)rrrrrrrrrrr=rrr1rT num_draftr rArBrar_r]r^rXr`r)rrrarr,_s        u           z&GPUModelRunner.propose_draft_token_ids overridescCsZddh}|D]"\}}||vsJd|d|t||}t||}t|||qdS)Nr&r"zConfig `z"` not supported. Allowed configs: )rrrsetattr)rrgallowed_config_names config_nameconfig_overridesconfig new_configrrrr s   zGPUModelRunner.update_configz Loading (GPU)) span_name eep_scale_upc Cstjd|jjdd|rt|jnd\}}}|jjr%t|j|j|_ d}zt }t }t |j}|j|j|jd|_|jrK||j|j|j|_t|drtd|j|jt|jd rt|jjr|jjr|jj} | d usvJ| jd us}Jtd | jj|r||nd } |r||nd } |j d urt|j|j|_ |j |jj| j| | ||d 7}|jrt|std |} | rtd| n|j} |j | t } Wd n1swY|j!|_"Wn t#j$j%y}zd}|d|d}t&||d }~wwtjdt'|j"| |ddt(|jt)|dd }r:t)|d d }r:t(||jj*}t+|oM|d uoM|,|_,t|jr|jjrtd|jj|ri||nd } |rr||nd } |j d us|J|j |j|j| | ||j j-r|j j.|d|jj/j0t1j2kr|jj/3|j}t4j5d 7_5|jj6d|dd S|j/j7}|d usJ|8r|jj9st:|j|jt;j|j|_d Sd S)z_ Args: eep_scale_up: the model loading is for elastic EP scale up. zStarting to load model %s...globalscope)NNNr)rr"r]zLoading drafter model...r)Nz%EPLB is enabled for drafter model %s.rzRModel does not support EAGLE3 interface but aux_hidden_state_outputs was requestedz2Using auxiliary layers from speculative config: %szFailed to load model - not enough GPU memory. Try lowering --gpu-memory-utilization to free memory for weights, increasing --tensor-parallel-size, or using --quantization. See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ for more tips.z (original error: )z1Model loading took %s GiB memory and %.6f secondslocalzEPLB is enabled for model %s.) rank_mappingT) fullgraphr)r)?r: info_oncer"r)r get_eep_stater'rrrSrTtime perf_counterr2r& load_modelrr%load_lora_modelrr]r9r)re add_modelrYr:rr "_get_eagle3_aux_layers_from_configinfo"get_eagle3_aux_hidden_state_layersset_aux_hidden_state_layersconsumed_memorymodel_memory_usagerrOutOfMemoryErrorrrUr%rmultimodal_configr<r2is_asyncstart_async_loopr$rrr init_backendrstock_torch_compile_countcompilerrhas_full_cudagraphsrrrrrrs)rroglobal_expert_loads#old_global_expert_indices_per_modelru eplb_modelsmtime_before_load model_loaderr=global_expert_loadold_global_expert_indices aux_layerstime_after_loademsg combined_msgr] drafter_model mm_configrrrrrrr{s               E          zGPUModelRunner.load_modelcCsN|jr|jjs dS|jjj}t|dsdS|j}|r%t|ttfr%t|SdS)amExtract Eagle3 auxiliary layer indices from speculative config. These indices specify which hidden states from the base model should be used as auxiliary inputs for the Eagle3 drafter model during speculative decoding. Returns: Tuple of layer indices if found in draft model config, None otherwise. N eagle_aux_hidden_state_layer_ids)r)rerrrrrrk)rr layer_idsrrrr~s   z1GPUModelRunner._get_eagle3_aux_layers_from_configweights_iterator weights_pathis_checkpoint_formatcCs|dur |s td|}dd|D}t}|durOt|j}t|ds4t d|jj d|dur<||j _ | |j |}tttttjf|}tjdd d |jjjp_|jjj}t|:|rxt|||} t||j n tjd d d t} |D]\} } || } | | | | qWdn1swYt} | |}tjd |d d |j j dur| dur|| }|rtd |dSdSdSdS)a Reload weights from a weights iterator or from disk :param weights_iterator: weights to load into model :param weights_path: path to load weights from if weights_iterator is not provided. Use path of original model if neither is provided. :param is_checkpoint_format: set to False if weights have already been processed into kernel format (repacking, renaming, ect.) NzReloading from disk means that weights will be in checkpoint format. Please use `is_checkpoint_format=True` to avoid weight reloading errorscSh|]\}}|qSrr)rrrTrrr r\z0GPUModelRunner.reload_weights..get_all_weightszModel reloading with `z` formatzReloading weights inplace...rtrqzhReloading with `is_checkpoint_format=True` requires that weights be in kernel format and already shardedz2Reloading and processing weights took %.2f secondsz5Following weights were not loaded from checkpoint: %s)!r:warningrnamed_parametersryrzr2r&rNotImplementedError load_formatr"r)rrrrkrrrrwrr device_configr4 load_weightsr3 warning_oncer get_parameterrjr quantization)rrrrr)weights_to_loadcounter_before_reloadingr load_deviceloaded_weightsr loaded_weightrcounter_after_reloading diff_secondsweights_not_loadedrrrreload_weightssl         zGPUModelRunner.reload_weightstensorizer_configrcCstj|||jddS)N)rr")r1 save_modelrr")rrrrrsave_tensorized_model/s  z$GPUModelRunner.save_tensorized_modelcCs|j}|siS|jj}i}g}|D]\}}||} | dur!q|j|} | jdur,qt| j} t | jj |j dd} ||} | sRt | d|d} | ||<| j}|d}| |}| |krd| }n |}||| ||<|dkrtq|jj|}|jj|}||||}|j|}| |||}|j|}|j|||\}}}}t|||}| j|j|dd| j|j|dd| j|j|ddq|D]}||=||=q|r||S)NTrrr)rhroin_progress_prompt_logprobs_cpurrrgrr;rrkrrrx empty_cpurrrrrrir)rrRcompute_logprobsgather_logprobsslicelogprob_token_idsrjrselected_token_ranksr)rrrnum_prompt_logprobs_dictin_progress_dictrcompleted_prefill_reqsrrhrr9rrrr` start_toknum_remaining_tokensrrrTprompt_hidden_statesr tgt_token_idsrrr<rT chunk_slicerrrr9sp              z(GPUModelRunner._get_prompt_logprobs_dictcCszA|durdd|jjDWSi}|jdd}|jjD]}|jj|}|dur:||jdkr:t||nd||<q!|WSt yLiYSw)NcSsi|]}|dqSrr)rrrrrrrz6GPUModelRunner._get_nans_in_logits..rr;r) rorisnanrrrrrr IndexError)rrrnum_nans_for_indexrr/rrrrs    z"GPUModelRunner._get_nans_in_logitsc#sjjj}tjo |dk}|sdVdS|durCtjdtjffdd }t d|j |d| ddd dV| ddStjdtjffd d }|dusVJt d |j |d| ddd dV| ddS) z Randomize input_ids if VLLM_RANDOMIZE_DP_DUMMY_INPUTS is set. This is to help balance expert-selection - during profile_run - during DP rank dummy run rNrcstjjjdjdS)Nr)lowhigh)r randint_liker}rr"rmrrrrrand_input_idss z=GPUModelRunner.maybe_randomize_inputs..rand_input_idsz'Randomizing dummy input_ids for DP RankrTrcstjjSr)r randn_likerrrrrrrand_inputs_embedsszAGPUModelRunner.maybe_randomize_inputs..rand_inputs_embedsz+Randomizing dummy inputs_embeds for DP Rank)rr'rrVLLM_RANDOMIZE_DP_DUMMY_INPUTS functoolscacherrr:rtrjrr)rr}rdp_sizerandomize_inputsrrrrrmaybe_randomize_inputss*     z%GPUModelRunner.maybe_randomize_inputsrNmax_items_per_batchcCst|jdusJ|jj|j|di|jjd}|d|d}|dus%Jdtddt||fg||j|jd DS) z.rL) rrHget_dummy_mm_inputsr"rr8rHrr)rrNrdummy_mm_inputs dummy_mm_itemrrrrZs z"GPUModelRunner._get_mm_dummy_batchrforce_attentionr skip_eplbcreate_mixed_batch remove_lorais_graph_capturingrc 0 Cs|jjj} | r| jrtgtgfS|dus|sJ|r#|jn|} ||jj ks-J|jj }|rS|r7Jt |d|d}||}|d}dg||g}|} n8|rs|rYJt |t || }| g|}|| dkrr|| |d<nt ||}||}|g|}|d||7<t ||ksJt||ksJtj|tjd}t| }tj|tjd}|j|||| d||p|tjk|| dk| d \}}}}}|dur|}n||ksJd |d |d |j}|jdur|jn|}t|||||jjj\}}td ||d} |j||||d \}!}"|s|tjkrk|r'dg||dg}#n| }#|#|j jd|<d|j j|d<|j !|"|\}$}|$|j#jd|d<|j#!|tjk}%|j$||| |%rc|n|| |!d\} }|%|j&||| | M||j'ksJ|(}&|j)r|jj*s|+|\}'}(i|&|,|}&n|j-rd}'|j.j/d|}(|(}&n |j0j/d|}'d}(|j1r|j2j/ddd|f})n|j3dkr|j4j/ddd|f})n|j5j/d|})t6j7rd}*n|j8dur|j9j:|j'|jj;|j|'|(3t?| |j||||||"d|j9d|'|)|*|(d|&}+Wdn 1sKwYWdn 1s[wY|j@ri|+\},}n|+},|jAr|jABs{|jACrtD|jEtFtGBsJ|jAdusJ| o|tjHkp| o|tjko|jAjI }-|jJjKr| dkrd}-|jEjL||-| |"dWdn 1swY|M|s|jNd|dtO|d}.tP|.jQ|j 0. NrrErrrF) rrrrrrrrrrz7Cudagraph runtime mode mismatch in dummy_run. Expected z , but got rRrr)rrrrrr) batch_sizerr)rrrrrrr)use_cudagraphsrrT)rrrr)Rrr"rmm_encoder_onlyrrkvalid_runtime_modesrr(rr9rrRrr;rrr|rrrrrsrrrr'rr:r;rrrrhrcrrmaybe_dummy_run_with_lorar%r8rrLrMrr\r0rrr}rIrrJrrr!rrr)make_empty_intermediate_tensorsrrrrr'rYr)r_r^rr]rrr enforce_eagerr$cudagraph_specialize_lora dummy_runrrr^rr)0rrrrrrrrrrrrrrrnum_decode_tokensnum_prefill_tokensrnum_scheduled_tokens_listmin_tokens_per_reqrrr_cudagraph_moderrrrTrrrr!rr%rrcum_num_tokensr#rr}rrroutputsrr logit_indiceslogit_indices_devicerrrrs '                                 zGPUModelRunner._dummy_runc sjjj}|r|jrtgSt|}j|}| dfdd}t d'id|dddddd |d d || d d d iddddddd|dd|dd|ddddt Ddddt Ddddidt }z j ||d}Wnty}zdt|vrtd d!||d}~wwjrd"dt D}t|j} td#d$|D} d} tj| |jd%j|jd&}| | |||S)(Nrcstjf|jdS)Nr)rr@r)r{rrrrrbr\z3GPUModelRunner._dummy_sampler_run.. temperatureg? all_greedyF all_randomtop_pg?top_krrmax_num_logprobs no_penaltiesTrfrequency_penaltiesg?presence_penaltiesrepetition_penaltiesrcSrrrrrrrrqz5GPUModelRunner._dummy_sampler_run..spec_token_idscSrrrrrrrrrrallowed_token_ids_maskbad_words_token_idsrr out of memoryz9CUDA out of memory occurred when warming up sampler with m dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.cSsg|]}dgqSrrrrrrrrcss|]}t|VqdSr)r;)rrrrrrsz4GPUModelRunner._dummy_sampler_run..rr/r)rr"rrrrk rand_liker)rrrrrrRr rr)r make_dummyrrrandnrrrb) rrrr dummy_tensorsdummy_metadatarrrdummy_spec_decode_metadatar draft_probsrrr_dummy_sampler_runOs             z!GPUModelRunner._dummy_sampler_runrc CsR|jd}|jj}t||}||}t||}|d||7<t||ks+Jt||ks3J||}t |} tj ||ftj |j d} t t|} t|d} | |j| j|} | | t| | | g|ddt|Dd}|j|| |j dz| j||d WSty}zd t|vrtd |d |d ||d}~ww)Nrrr!)rcSrr)r~rrrrrrz9GPUModelRunner._dummy_pooler_run_task..)rrrpooling_states)rrrrz9CUDA out of memory occurred when warming up pooler (task=z) with r)rr(r9rrr@rr;rrrr|rrr@rrIverifyr"rr r r}rrr r)rrrrrrrrreq_num_tokensdummy_prompt_lensdummy_token_idsr)dummy_pooling_paramsr*rrrrr_dummy_pooler_run_tasks^         z%GPUModelRunner._dummy_pooler_run_taskcCs|jjj}|r|jrtgS|}|std|jjdt t t f}|D]}| ||}t dd|D||<~q(t|dddd}| ||S) NzModel zq does not support any pooling tasks. See https://docs.vllm.ai/en/latest/models/pooling_models.html to learn more.css|] }|dur|jVqdSr)nbytes)rorrrrsz3GPUModelRunner._dummy_pooler_run..cSs|dSrrrrrrrsz2GPUModelRunner._dummy_pooler_run..)keyr)rr"rrrrkrur r)rrMfloatrrrlr)rrrsupported_pooling_tasks output_sizerrmax_taskrrr_dummy_pooler_runs     z GPUModelRunner._dummy_pooler_runc Cs*|jre|jj}|dur|jrtdnQ|j}|dusJ|}dkre|js.tdn7| }|j |}td|||| ||}|j j d i|}t||dt|D] \}} | |jd|<qX|j|jdd\} } tjr|jr||| } n|| } nd} |~ ~ |jtdS) NzCSkipping memory profiling for multimodal encoder and encoder cache.rzfSkipping encoder profiling for embedding-only mode (all modality limits=0 with enable_mm_embeds=True).zxEncoder cache will be initialized with a budget of %s tokens, and profiled with %s %s items of the maximum feature size.r+tmp_T)rr)rLr"rskip_mm_profilingr:rrget_encoder_budgetrXrYmm_max_items_per_batchrZr)r9rrrXrr8r!rZrArr rrgccollect) rrrencoder_budgetr[max_mm_items_per_batchbatched_dummy_mm_inputsdummy_encoder_outputsrrrlast_hidden_statesrrr profile_runs`         zGPUModelRunner.profile_runz Capture modelc Cs |jjtjkrtddStjd7_t }t dd}t d|Bt |j d,tjd}|jD] \}}|j||dq:tjtjd}Wdn1s\wYWdn1skwYt d tt }||}||} tjd || d d d | S)NzrSkipping CUDA graph capture. To turn on CUDA graph capture, ensure `cudagraph_mode` was not manually set to `NONE`rrcss\ttj }|rtzdVW|r ttdSdS|r-ttwwr)rr rVLLM_ENABLE_CUDAGRAPH_GCfreezeunfreeze) should_freezerrr freeze_gcLs  z/GPUModelRunner.capture_model..freeze_gcTr)batch_descriptorsrFz4Graph capturing finished in %.0f secs, took %.2f GiBr rtrq)r$rrrrsr:rrnum_gpu_runner_capture_triggersryrzrrr#rrr mem_get_inforget_capture_descs_capture_cudagraphsrrrw) r start_timer+start_free_gpu_memoryr batch_descsend_free_gpu_memoryend_time elapsed_timecuda_graph_sizerrr capture_model?sJ  zGPUModelRunner.capture_modelr,c Cs|tjkr |sJd||sdS|dj}|tjk}tj|j|dd|d}tr?t ||j j d |r8dnd|j d }|D]7}|j}|j}|jjo\|tjko\|o\t|jj||d } t|jjD] } ||tj| |d qc|||| |dd qA||jdS) Nz Invalid cudagraph runtime mode: rTF)rrrrzCapturing CUDA graphs ({}, {})decodezmixed prefill-decode)disabledesc)rlrr)rrr)rrrr)rrsruniformrrpartialrr$rr&use_tqdm_on_loadformatrrrr'rrrrr$cudagraph_num_of_warmupsmaybe_remove_all_lorasr%) rr,rrrrrrrrrTrrrr0sl       z"GPUModelRunner._capture_cudagraphsrc stjdks JdGdddtdtdttttftt t fffdd }d tttfd t dtt fd d }g}g}|j D]}||}||d||d qG||j tjt|D] \}} j|| |qldS)zT Initialize the attention backends and attention metadata builders. rz*Attention backends are already initializedc@s"eZdZUeeed<eed<dS)zAGPUModelRunner.initialize_attn_backend..AttentionGroupKey attn_backendrN)rrrrrZrrnrrrrAttentionGroupKeys  rCkv_cache_group_specrc stttt}tj||j}itt}|jD]5}|| }|j vr*t d|}| }|j }t|tr;|j|}||f}|||<|||qfdd|DtddDfS)N FastPrefillcsi|] \}}||qSrrry attn_backendsrrrrz_GPUModelRunner.initialize_attn_backend..get_attn_backends_for_group..css|]}|jVqdSr)rB)r group_keyrrrrsz^GPUModelRunner.initialize_attn_backend..get_attn_backends_for_group..)rrr r-rrrrrget_attn_backendrrb full_cls_namerrrqrrrrr) rD layer_typelayersattn_backend_layersrrBrJlayer_kv_cache_specrrCrrFrget_attn_backends_for_groups2      zKGPUModelRunner.initialize_attn_backend..get_attn_backends_for_groupattn_backends_mapkv_cache_group_idcSs6g}|D]\\}}}t||||}||q|Sr)rrr)rQrRrWrBrrrrrrcreate_attn_groupss zBGPUModelRunner.initialize_attn_backend..create_attn_groupsrN)r;rWr rmrkrrrrrrZrrrr _check_and_update_cudagraph_moderrr) rrrPrSattention_backend_mapsattention_backend_listrDrGrattn_backend_maprrOrinitialize_attn_backends6%   z&GPUModelRunner.initialize_attn_backendrcCsjtt|jD]'}|j|D]}|j|j|j|t|kr ||nd|jjs'dn|jj dqq| dS)zW Create the metadata builders for all KV cache groups and attn groups. Nr)num_metadata_builders) rr;rrWcreate_metadata_buildersrrr'rr!calculate_reorder_batch_threshold)rrrrRrrrrinitialize_metadata_builders%s  z+GPUModelRunner.initialize_metadata_buildersattention_backendsrc Cstj}d}t||D] \}}|D]}|}||j|j} | j|jkr)| }|j}qq |j j } | dus5J| t j kr||tjkr|d| jd|d|d} |tjkr[| d7} t| |j rl| d7} t j} |j _ n | d7} t j} |j _ t| | t j kr|tjkrd| jd|d|d} |j jtjkr|j s|j jr| d 7} t j} |j _ n | d 7} t j} |j _ t| | t j kr|jd kr|jtjjkrd| jd |d |d} |j r| d7} t j} |j _ n | d7} t j} |j _ t| | r|tjkrtd| jd|d|d| t j krJ| rJ|jd krJ|j !|j|j"j#|j j$} | durG| ng|_%| |j _ |j&'| |j|j(rq|j()rst*|j+t,siJ|j+'| dSdSdS)z Resolve the cudagraph_mode when there are multiple attention groups with potential conflicting CUDA graph support. Then initialize the cudagraph_dispatcher based on the resolved cudagraph_mode. NzCUDAGraphMode.z is not supported with z backend (support: rszU; please try cudagraph_mode=PIECEWISE, and make sure compilation mode is VLLM_COMPILEz+; setting cudagraph_mode=FULL_AND_PIECEWISEz); setting cudagraph_mode=FULL_DECODE_ONLYzJ; setting cudagraph_mode=PIECEWISE because attention is compiled piecewisezI; setting cudagraph_mode=NONE because attention is not compiled piecewiserz9 is not supported with spec-decode for attention backend z (support: z"; setting cudagraph_mode=PIECEWISEz; setting cudagraph_mode=NONEz backend (support:zW) ; please try cudagraph_mode=PIECEWISE, and make sure compilation mode is VLLM_COMPILE)-r[ALWAYSrget_builder_clsget_cudagraph_supportrrrrr$rr mixed_moderrrNEVERrasplitting_ops_contain_attentionFULL_AND_PIECEWISEFULL_DECODE_ONLYr:r decode_moderr VLLM_COMPILEuse_inductor_graph_partitionrrsr UNIFORM_BATCHrseparate_routine&adjust_cudagraph_sizes_for_spec_decoder'rrqrurinitialize_cudagraph_keysr)r_rr]r) rr]rmin_cg_supportmin_cg_backend_nameattn_backend_setrrB builder_cls cg_supportrrr capture_sizesrrrrT<s                       z/GPUModelRunner._check_and_update_cudagraph_modecCs@dd}dd|D}t|dkrd|_dSt|||_dS)a Choose the minimum reorder batch threshold from all attention groups. Backends should be able to support lower threshold then what they request just may have a performance penalty due to that backend treating decodes as prefills. cSs"|dur|S|dur |St||Sr)r)abrrrrs"zBGPUModelRunner.calculate_reorder_batch_threshold..cSsg|]}|jqSr)rrrrGrrrrzDGPUModelRunner.calculate_reorder_batch_threshold..rN)_attn_group_iteratorr;rr )r min_none_highreorder_batch_thresholdsrrrr[s z0GPUModelRunner.calculate_reorder_batch_thresholdkv_manager_block_sizerWcCsdtttdtdtfdd}dd|D}|||r|Stdd |D}t|d d D]}||d kr5q,|||r>|Sq,td |d)a Select a block size that is supported by all backends and is a factor of kv_manager_block_size. If kv_manager_block_size is supported by all backends, return it directly. Otherwise, return the max supported size. Args: kv_manager_block_size: Block size of KV cache attn_groups: List of attention groups Returns: The selected block size Raises: ValueError: If no valid block size found backendsrnrcSsl|D]1}d}|D]#}t|tr||krd}q t|tr'||jdkr&d}q td||s3dSqdS)zO Check if the block size is supported by all backends. FTrzUnknown supported size: ) get_supported_kernel_block_sizesrrr`basera)r{rnr is_supportedsupported_sizerrrblock_size_is_supporteds    zHGPUModelRunner.select_common_block_size..block_size_is_supportedcSsg|]}|jqSr)rrurrrrrz;GPUModelRunner.select_common_block_size..css,|]}|D] }t|tr|VqqdSr)r|rr)rrrrrrr*s z:GPUModelRunner.select_common_block_size..T)reverserzNo common block size for z. )rrrZrrrrtra)rzrWrr{all_int_supported_sizesrrrrselect_common_block_sizes(      z'GPUModelRunner.select_common_block_sizec Cs dd|jD}g}t|j|j}t|jD]1\}}t|jtr!qt|||t }t|jt rB|j j r7|nd|jj } t|| }||q||j jgksV||j jgkr|j jdks`Jdt|j||j|j|j|j|||t|jj|jj|jj|jd |_dSdS)a] Re-initialize the input batch if the block sizes are different from `[self.cache_config.block_size]`. This usually happens when there are multiple KV cache groups. Args: kv_cache_config: The KV cache configuration. kernel_block_sizes: The kernel block sizes for each KV cache group. cSs g|] }t|jts|jjqSr)rrrjrn)rrrrrrDs  z?GPUModelRunner.may_reinitialize_input_batch..rrzCannot re-initialize the input batch when CPU weight offloading is enabled. See https://github.com/vllm-project/vllm/pull/18298 for more details.) rrrrrrrrmax_num_blocks_per_reqrrrrAN)rrlrrOrrrrjrRrror#enable_prefix_cachingnum_speculative_blocksrrnr,rrr8rrr"rmrrr)rorrrA) rrrrmax_num_blocksrrrrmamba_blocks_per_reqrrrmay_reinitialize_input_batch8sV      z+GPUModelRunner.may_reinitialize_input_batchcCsi}|jD]}tj|jtj|jd}|jD]}|||<qqt}|jD]}|j D] }||j vr0q(| |q(q#|t| ksCJd|S)au Initializes the KV cache buffer with the correct size. The buffer needs to be reshaped to the desired shape before being used by the models. Args: kv_cache_config: The KV cache config Returns: dict[str, torch.Tensor]: A map between layer names to their corresponding memory buffer for KV cache. r!z)Some layers are not correctly initialized) kv_cache_tensorsrrrint8r shared_byrrrrrr)rrkv_cache_raw_tensorskv_cache_tensorrkrrrGrrr_allocate_kv_cache_tensorsts&        z)GPUModelRunner._allocate_kv_cache_tensorscCstj|jSr) itertoolschain from_iterablerWrrrrrwrz#GPUModelRunner._attn_group_iteratorccs(|jjsdS|jD]}|EdHq dSr)rrrW)rrWrrr"_kv_cache_spec_attn_group_iterators   z1GPUModelRunner._kv_cache_spec_attn_group_iteratorc Csg}t|jD]H\}}|j}t|trtt|j}t|t r"qt|t r<|j |}|jj }| ||}||qt|trH||j qtd|j|S)a Generate kernel_block_sizes that matches each block_size. For attention backends that support virtual block splitting, use the supported block sizes from the backend. For other backends (like Mamba), use the same block size (no splitting). Args: kv_cache_config: The KV cache configuration. Returns: list[int]: List of kernel block sizes for each cache group. zunknown kv cache spec )rrrrrqr8iterrrrjrgrWrnrrror) rrrrrrrWrzselected_kernel_sizerrr_prepare_kernel_block_sizess(       z*GPUModelRunner._prepare_kernel_block_sizesrc sDi}d\}}|D] }|j}|j} |jt|krq ||j} |jD]} | |jvr+q#|| } | |jdks:J| |j} t |t rd}|j | }| |}| j || |j |j|jjd|j}z| ttksqJWnttfytttYnwtfddDfddttD}|| |j||| <q#t |trd}|| } g}d}t|j|jD]G\}}t|}|j|}| g|R}t| }|g|d d R}||dksJtj!| |||||d }|"|||d|7}q||| <q#tq |r |r |#||S) a Reshape the KV cache tensors to the desired shape and dtype. Args: kv_cache_config: The KV cache config kv_cache_raw_tensors: The KV cache buffer of each layer, with correct size but uninitialized shape. kernel_block_sizes: The kernel block sizes for each KV cache group. Returns: Dict[str, torch.Tensor]: A map between layer names to their corresponding memory buffer for KV cache. FFrT)cache_dtype_strc3s|]}|VqdSrrr)kv_cache_shaperrrrz;GPUModelRunner._reshape_kv_cache_tensors..csg|]}|qSr)rer)kv_cache_stride_orderrrrrvz.rN)rstridestorage_offset)$rrrrRr;rrrpage_size_bytesrrgrnget_kv_cache_shaper head_sizer#r-rget_kv_cache_stride_orderAttributeErrorrrkrviewpermuterorshapesdtypesrXrrr as_stridedr%_update_hybrid_attention_mamba_layout)rrrrrThas_attn has_mambarGrrBkernel_block_sizer raw_tensor num_blocksnum_blocks_per_kv_blockkernel_num_blocksr inv_order state_tensorsstorage_offset_bytesr dtype_sizenum_element_per_page target_shaper target_striderkr)rrr_reshape_kv_cache_tensorss            H z(GPUModelRunner._reshape_kv_cache_tensorsrTcCs|D]H}|j}|jD]?}||}t|trK|jddkrK|jddks-Jd|j|jdd}|j|j|d|g|ddRdq qdS)z Update the layout of attention layers from (2, num_blocks, ...) to (num_blocks, 2, ...). Args: kv_caches: The KV cache buffer of each layer. rrErzkFail to determine whether the layout is (2, num_blocks, ...) or (num_blocks, 2, ...) for a tensor of shape N)rr) rrrrrgrr as_strided_r)rrTrGrrkv_cache hidden_sizerrrr0s$  z4GPUModelRunner._update_hybrid_attention_mamba_layoutc Cs|jj}||j|r |||j||j|\}}}||_||_n ||}| |||}|j D]\}} t d|| || ||<q1|jjjdkrLdnd} t||jj|j| |S)a\ Initialize the memory buffer for KV cache. Args: kv_cache_config: The KV cache config kernel_block_sizes: The kernel block sizes for each KV cache group. Returns: Dict[str, torch.Tensor]: A map between layer names to their corresponding memory buffer for KV cache. z%s reuses KV cache of %s longcat_flashrEr)r#r-use_uniform_kv_cacherWallocate_uniform_kv_cachesrrUrVrrrrr:r;r"r model_typerr$rrT) rrrr-rTrUrBrrtarget_layer_namenum_attn_modulerrrinitialize_kv_cache_tensorsKs8  z*GPUModelRunner.initialize_kv_cache_tensorscCsb|jsdSt|j|j|j|jjr-t|jt}t |D]}||jvr*|j |qdSdSdS)z Add layers that re-use KV cache to KV cache group of its target layer. Mapping of KV cache tensors happens in `initialize_kv_cache_tensors()` N) rrrrr#rrrr+reversedrr)rrrrrrr.maybe_add_kv_sharing_layers_to_kv_cache_groupss   z=GPUModelRunner.maybe_add_kv_sharing_layers_to_kv_cache_groupscCst|}||_|||||||}|||||||||}|j rI|j s9|j rIt |j ttBsCJ|j |trnt}|jdurd|jdus[J||j|jn|||t|jjrx|dSdS)z Initialize KV cache based on `kv_cache_config`. Args: kv_cache_config: Configuration for the KV cache, including the KV cache size of each layer N)rr.may_add_encoder_only_layers_to_kv_cache_configrrXrr\rrr)r_r^rr]rrvalidate_same_kv_cache_grouprrrUrVregister_cross_layers_kv_cacheregister_kv_cachesset_host_xfer_buffer_opsrr"rinit_routed_experts_capturer)rrrrTkv_transfer_grouprrrinitialize_kv_caches<          z"GPUModelRunner.initialize_kv_cachecCsdtd|jjt}|jj}|jj t |jj d||_ |j |jj|j |jd||dS)NzFInitializing routed experts capturer, enable_return_routed_experts: %sr)rmax_num_kv_tokensr)r:rr"rr.creater#rnrrr;rr init_bufferr(rr_bind_routed_experts_capturer)rrouted_experts_capturerrnrrrrs"z+GPUModelRunner.init_routed_experts_capturerrcCsdddlm}ddlm}|jjD]}t||r/t|j|r/|j }||fdd}|j |qdS)Nr)FusedMoE) BaseRoutercSs|||dSr)capture)topk_ids _layer_id _capturerrrr _capture_fnszAGPUModelRunner._bind_routed_experts_capturer.._capture_fn) *vllm.model_executor.layers.fused_moe.layerr7vllm.model_executor.layers.fused_moe.router.base_routerrr$rrrrouterlayer_idset_capture_fn)rrrrrrrrrrrs   z,GPUModelRunner._bind_routed_experts_capturerc Cs|jjj}tt}t|jt}|D]"\}}|jt j kr5t ||j |j |jd}||||j|qt|dkrYt|dksFJd|\}}|jjt||ddSdS)zA Add encoder-only layers to the KV cache config. )rnrrrrrz0Only support one encoder-only attention spec now)rrN)rr#rnrrrr+r attn_typer^ ENCODER_ONLYrjrrr.rrrr;popitemrrrm) rrnencoder_only_attn_specsrr attn_module attn_specrrrrrrs.       z=GPUModelRunner.may_add_encoder_only_layers_to_kv_cache_configcCs~tr tjr iSi}tttt}t|j|}| D] \}}t |t r0|j }r0||j |<q||j}r<|||<q|S)a0 Generates the KVCacheSpec by parsing the kv cache format from each Attention module in the static forward context. Returns: KVCacheSpec: A dictionary mapping layer names to their KV cache format. Layers that do not need KV cache are not included. )rrrrrr r-rrrrr+kv_sharing_target_layer_namerget_kv_cache_spec)rrrKrrr kv_tgt_layerrrrrr s   z GPUModelRunner.get_kv_cache_speccCs>|jd|jd}|j|dd|j|j|SrQ)rrrjrrrr)rrpinnedrrrr,s    zGPUModelRunner._to_listcCsL|jdd|jD}|j|WdS1swYdS)z Get encoder timing stats for all requests and clear the registry. Returns: Dictionary mapping request_id to stats dict. cSsi|] \}}||qSr)to_dict)rr stats_objrrrrCrz;GPUModelRunner.get_encoder_timing_stats..N)rzrwrr)rstatsrrrget_encoder_timing_stats;s $z'GPUModelRunner.get_encoder_timing_statsr=group_lora_refsrHrIc csz|sdVdS||||}dd|D}tjt}zQdVWtjt|}|tt|d} |j,|D] } | |jvrLt |j| <|j| } | j | 7_ | j d7_ q?WddS1skwYdStjt|}|tt|d} |j+|D] } | |jvrt |j| <|j| } | j | 7_ | j d7_ qWdw1swYw)a@ Context manager to time encoder forward operations. Args: should_time: Whether timing is enabled group_lora_refs: Full list of (request_id, pos_info) tuples current_item_idx: Starting index for this group num_items: Number of items in this group NcSrrr)rrrTrrrr`r\z9GPUModelRunner.timed_encoder_operation..r) rrrryrzrlr;rzrwEncoderTimingStatsencoder_forward_timenum_encoder_calls) rr=rrHrI group_refsgroup_request_idsr1elapsedper_request_timerrrrrr7JsD      "    z&GPUModelRunner.timed_encoder_operation)rN)rrrNr)F) NNNNFFNNN)rrrr)NNNN)TFNNNr)NNT)rrrN) NFFTFFFTFr)rrrrrrrrrrrinference_moderr rSymIntrrrr{rrrvrr:rrFrrerrr r rErVr\rndarrayrkrcrrrnrrrrrrr_rrr]rrrrrrFrGr%rPrhnnModulerrLrorMrurNrrrKrrrvryrsrrrrurr{rrwrxrrrr staticmethodrrr&rrrrr*rDr5r3rtrPr-rr8rrr,rrOr{r~rrrrrrrZrr rzrrr&r8r0rlrXr\rrrZrmrTr[rrrrrrwrrrrrrrrr.rrrrrrr7rrrrrIs   +  "   x 0 "    ~  - a      u $ a 1/ O  +   m   <   t              })" J 6  C  "!     "  - - T    i   ,      O N 7 KF NQ!G<  +  h  7 3 &rc@sHeZdZUdZdZeed< dZeed< de e eeBffddZ d S) rz7Per-request timing statistics for encoder forward pass.grrrrcCs|j|jdS)NrrrrrrrrszEncoderTimingStats.to_dictN) rrrrrrrrrrrrrrrrrws   r(rrrrxry collectionsrcollections.abcrrr contextlibrrr dataclassesr r typingr r r rrrrrtorch.distributedtorch.nnrr vllm.envsrvllm.compilation.counterrvllm.compilation.cuda_graphrrvllm.compilation.monitorr vllm.configrrrrrvllm.distributed.ec_transferrr vllm.distributed.eplb.eplb_statervllm.distributed.kv_transferrr/vllm.distributed.kv_transfer.kv_connector.utilsrvllm.distributed.parallel_stater r!r"r#r$r%vllm.forward_contextr&r' vllm.loggerr(vllm.lora.layersr)r*$vllm.model_executor.layers.attentionr+r,/vllm.model_executor.layers.attention_layer_baser-r?*vllm.model_executor.models.interfaces_baser@rArBvllm.multimodalrCvllm.multimodal.encoder_budgetrDvllm.multimodal.inputsrErFrGvllm.multimodal.utilsrHvllm.pooling_paramsrIvllm.sampling_paramsrJ vllm.sequencerK vllm.tasksrLrMrN vllm.tracingrO vllm.utilsrPvllm.utils.jsontreerQvllm.utils.math_utilsrRrSvllm.utils.mem_utilsrTrUvllm.utils.nvtx_pytorch_hooksrVvllm.utils.platform_utilsrWvllm.utils.torch_utilsrXrYvllm.v1.attention.backendrZr[r\r]r^r_r`#vllm.v1.attention.backends.gdn_attnra vllm.v1.attention.backends.utilsrbrcrdvllm.v1.core.sched.outputrevllm.v1.cudagraph_dispatcherrfvllm.v1.kv_cache_interfacergrhrirjrkrlrmrnrorprqvllm.v1.outputsrrrsrtrurvrwrxryrzr{r|vllm.v1.pool.metadatar}r~vllm.v1.sample.logits_processorrr)vllm.v1.sample.logits_processor.interfacervllm.v1.sample.metadatar vllm.v1.sample.rejection_samplerrvllm.v1.sample.samplerrvllm.v1.spec_decode.draft_modelrvllm.v1.spec_decode.eaglervllm.v1.spec_decode.medusarvllm.v1.spec_decode.metadatar#vllm.v1.spec_decode.suffix_decodingrvllm.v1.structured_output.utilsr vllm.v1.utilsrrvllm.v1.workerrvllm.v1.worker.cp_utilsrrvllm.v1.worker.dp_utilsr.vllm.v1.worker.ec_connector_model_runner_mixinrvllm.v1.worker.gpu_input_batchrr!vllm.v1.worker.gpu_ubatch_wrapperr.vllm.v1.worker.kv_connector_model_runner_mixinr&vllm.v1.worker.lora_model_runner_mixinrvllm.v1.worker.ubatch_utilsrrrrvllm.v1.worker.utilsrvllm.v1.worker.workspacerutilsrrrr+vllm.model_executor.model_loader.tensorizerrrrr\rrr:rrrrrrrrrrrrrrrs&                4           $   44                      D* \