o .ihd@sUddlZddlZddlZddlZddlZddlmZmZddl m Z ddl m Z m Z mZddlZddlm ZddlZddlmZddlmZddlmZmZddlmZe rdddlmZdd lm Z ne!Ze!Z ddl"Z"e"#e$Z%ej&ej'ej(ej)ej*ej+ej+ej+ej,ej-ej+d Z.ej(ej(ej&ej&ej/ej/ej+ej+ej0ej0ej1ej1iZ2d d iZ3ed Z4dej5de6fddZ7ej8dej9fddZ:ej8de;fddZdej9de;fddZ?dej9dej9fd d!Z@d"eej9fd#d$ZAd%ej5d&e*d'e*ddfd(d)ZB dd*eCej9BdBd+eCej9BdBdej9fd,d-ZDd.eEeCe fdeCdBfd/d0ZFd.eEeCe fdej9dBfd1d2ZGd3eCd4edeCfd5d6ZHd3eCd4edej9fd7d8ZId9e;dBddfd:d;ZJ   < =dd>e;d?e;d@e;dAe;dBe;d*eCej9BdBd+eCej9BdBd9e;dBdCeCdBdDeCdBdeKeLej5eLej5ffdEdFZM  
e;d?e;d@e;dAe;dBe;d*eCej9BdBd+eCej9BdBd9e;dBdCeCdBdeKeLej5eLej5ffdGdHZNdIeLdej9dJeCejOBdKe6dej5f dLdMZPddNdOeLeLe4dPe4dejQdQe;dBdejRf dRdSZSdddTdUdOeLeLe4dPe4dej9dQe;dBdCeCejOBdBdKe6dej5fdVdWZTejUjVZWeXZYdXejUjZddfdYdZZ[e[ejU_VGd[d\d\Z\dejUjZfd]d^Z]da^ejUjZdBe_d_<dejUjZdBfd`daZ`e dbdcdddeCdBde;fdedfZade;fdgdhZbd%e de fdidjZcdkej5eLej5BeKej5Be Bdej5eLe BeKe Be BfdldmZddnej5dej5fdodpZedqeCdreCde6fdsdtZfdreCde6fdudvZgdreCde6fdwdxZhdreCde6fdydzZide6fd{d|Zjed}d~Zk     ddeCdedeLeCdBdedBdedBdeCdBdeKejldffddZmdS)N)Callable Collection) lru_cache) TYPE_CHECKINGAnyTypeVar)version)Version)Library infer_schema) ModelConfigIntermediateTensors) float32halffloat16bfloat16floatfp8fp8_e4m3fp8_e5m2int8fp8_inc fp8_ds_mlarrTtreturncCsZ|sdS|j}|}d}tt|dddD]}|||kr$dS|||9}qdS)a Check if tensor is contiguous AND has no degenerate strides. A degenerate stride occurs when a dimension has size 1 but the stride doesn't match the canonical contiguous layout. This can cause issues in some CUDA kernels that rely on stride values for memory access. For a C-contiguous tensor of shape (d0, d1, ..., dn), the expected strides are: stride[i] = product(shape[i+1:]) for all i, with stride[-1]=1. Example with torch.Size([16, 1, 8, 32]): - Canonical strides: (256, 256, 32, 1) - Degenerate strides: (256, 1, 32, 1) # dim=1 has size=1, allowing # non-canonical stride in dim=0 FT) is_contiguousshapestriderangelen)rr stridesexpected_strideir'S/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/vllm/utils/torch_utils.pyis_strictly_contiguousCs r)dtypecc(t}t|dVt|dS)z0Sets the default torch dtype to the given dtype.N)torchget_default_dtypeset_default_dtype)r* old_dtyper'r'r(set_default_torch_dtypea  r0 num_threadsccr+)zBSets the default number of threads for PyTorch to the given value.N)r,get_num_threadsset_num_threads)r2old_num_threadsr'r'r(set_default_torch_num_threadsjr1r6c csddlm}|sdVdStjd}dtjd<z6zdVWnty?}zdt|vr2d}nt|}t||d}~wwW|durKtjd=dS|tjd<dS|dur[tjd=w|tjd<w)z%Avoid unexpected CUDA initialization.rcurrent_platformNCUDA_VISIBLE_DEVICESzNo CUDA GPUs are availablezCUDA initialization is blocked.) vllm.platformsr8is_cudaosenvironget Exceptionstr RuntimeError)r8 old_valueeerr_msgr'r'r(guard_cuda_initializationss.       rFcCstjg|dS)z'Get the size of the data type in bytes.r*)r,tensor element_sizerGr'r'r(get_dtype_sizesrJcCs|tjk|j|jdS)N)r,boolis_floating_point is_complexrGr'r'r(_get_precision_levelsrO src_dtype tgt_dtypecCs||krdSt|}t|}||krdS||krdS|js6|js6t|}t|}|j|jko5|j|jkSt|}t|}|j|jkoQ|j|jkoQ|j|jkS)z[ Test whether it is lossless to cast a tensor from `src_dtype` to `tgt_dtype`. TF) rOrMrNr,iinfominmaxfinfo resolution)rPrQ src_level tgt_levelsrc_infotgt_infor'r'r(is_lossless_casts&        r[dtypescstfdddS)zv Get the common `dtype` where all of the other `dtypes` can be cast to it without losing any information. cstfddDS)Nc3s|]}t|VqdSN)r[).0dtrGr'r( sz?common_broadcastable_dtype....)sumrGr\rGr(sz,common_broadcastable_dtype..)key)rTrbr'rbr(common_broadcastable_dtypes rerHlowhighcCs:ddlm}tj|tjd}||||||~dS)Nr) _custom_opsrG)vllmrhr, empty_likeruniform_ convert_fp8)rHrfrgops tensor_tmpr'r'r(_generate_random_fp8s   ro cache_dtype model_dtypecCst|tr:|dkr)t|tr|tvrt|}|St|tjr"|}|Std||tvr3t|}|Std|t|tjrD|}|Std|)NautozInvalid model dtype: zInvalid kv cache dtype: ) isinstancerASTR_DTYPE_TO_TORCH_DTYPEr,r* ValueError)rprq torch_dtyper'r'r(get_kv_cache_torch_dtypes"    rw quant_cfgcCs|dd}|drn|d|}|dp$|dp$|dp$|d}t|trP|ddurB|d d krB|d d krBd }ntd|ttdSt|t rn| }|tvrat|Std|ttdSdS)aGet the KV cache quantization algorithm string from the quantization config. Maps various FP8 format names to vLLM's standard cache dtype strings. Returns None if no kv_cache_quant_algo is specified. Returns "auto" if the value is not recognized/supported. quant_methodr:modelopt quantizationkv_cache_schemekv_cache_quant_algodynamicFnum_bitstyperrzhWARNING: Unknown kv_cache_quant_algo '%s' in model config. Supported values: %s. Falling back to 'auto'.rrN) r? startswithrsdictloggerwarninglist#MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAPkeysrAlower)rxryquantization_innerkv_algo kv_algo_lowerr'r'r(get_kv_cache_quant_algo_strings@       rcCs$t|}|dur|dkrt|SdS)zKGet the KV cache quantization algorithm dtype from the quantization config.Nrr)rrt)rx kv_algo_strr'r'r(get_kv_cache_quant_algo_dtype$srkv_cache_dtype model_configcCsL|dkr|St|dd}|dur$t|dd}|dur$t|}|dur$|SdS)z}Resolve 'auto' kv_cache_dtype to the actual string value from model config. Returns the resolved cache_dtype string. rr hf_configNquantization_config)getattrr)rrhf_cfgrxrr'r'r(resolve_kv_cache_dtype_string-s  rcCs |dkr |r |jStjSt|S)Nrr)r*r,rrt)rrr'r'r(kv_cache_dtype_str_to_dtypeBsrseedcCsJ|dur!t|tj|t|tjr#tj|dSdSdSr])randomrnpr, manual_seedcuda is_availablemanual_seed_all)rr'r'r(set_random_seedKs    rrNHD num_blocks block_size num_layers num_heads head_sizedevice cache_layoutc st|t||} |d|||f| dvsJ| dkrdnd} tfdd| D} |d} g}g}t|D]B}tj| | |d j| }|d vrN|| | n|d krZt|| | nt d || |ddd f| |dddfq5||fS)NrK)rHNDr)rrrK)rrrrKrc3s|]}|VqdSr]r')r^r&generic_kv_cache_shaper'r(r`gz5create_kv_caches_with_random_flash..࿩sizer*rrrrrrr#Does not support key cache of type rr) rrwtupler"r,emptypermuterkroruappend)rrrrrrprqrrrr* stride_orderkv_cache_allocation_shapescale key_caches value_caches_key_value_cacher'rr("create_kv_caches_with_random_flashTs.   rc CsH|dkr|drtd|t|t||} |d} dtjg| d} |||| || f} g} t|D].}tj| | |d}|dvrL|| | n|dkrXt || | ntd|| |q6||||f}g}t|D].}tj|| |d}|dvr|| | n|dkrt || | ntd || |qq| |fS) Nrz6Does not support key cache of type fp8 with head_size rrGrrrz%Does not support value cache of type ) rurrwr,rHrIr"rrkror)rrrrrrprqrrr*rxkey_cache_shaperr key_cachevalue_cache_shaper value_cacher'r'r(create_kv_caches_with_random|s:       rdata target_device pin_memorycCs tj|||dd}|j|ddS)z?Asynchronously create a tensor and copy it from host to device.cpu)r*rrT)r non_blocking)r,rHto)rr*rrrr'r'r(async_tensor_h2dsrmax_lenrpadrcCsl|dur ttt|dd}tjt||f||d}t|D]\}}t||ks)J|||dt|f<q|S)z Make a padded array from 2D inputs. The padding is applied to the end of each inner list until it reaches `max_len`. Nr)defaultrG)rTmapr#rfull enumerate)rrr*rpadded_xindblocktbr'r'r(make_ndarray_with_pads rF)rrrc Cs8t|}t||||d}t||}|r|}|S)z Make a padded tensor from 2D inputs. The padding is applied to the end of each inner list until it reaches `max_len`. r)TORCH_DTYPE_TO_NUMPY_DTYPErr, from_numpyrr) rrr*rrrnp_dtyperrHr'r'r(make_tensor_with_pads rstreamcCs|t_t|dSr])_current_stream_tlsvalueprev_set_stream)rr'r'r(_patched_set_streams rc@seZdZddZdS)_StreamPlaceholdercCsdd|_dS)NcSsdSr]r'r'r'r'r(rcsz-_StreamPlaceholder.__init__..) synchronize)selfr'r'r(__init__sz_StreamPlaceholder.__init__N)__name__ __module__ __qualname__rr'r'r'r(rs rcCsddlm}ttdrtjdurA|s|r$tj tj tjS| r/t t_tjS|j }|dur=|t_tjStdtjS)a replace `torch.cuda.current_stream()` with `vllm.utils.current_stream()`. it turns out that `torch.cuda.current_stream()` is quite expensive, as it will construct a new stream object at each call. here we patch `torch.cuda.set_stream` to keep track of the current stream directly, so that we can avoid calling `torch.cuda.current_stream()`. the underlying hypothesis is that we do not call `torch._C._cuda_setStream` from C/C++ code. rr7rNzZFail to set current stream, current platform may not support current_stream with torch API)r;r8hasattrrris_rocmr<r,r set_streamStreamis_cpurcurrent_streamru)r8rr'r'r(rs    r _aux_streamcCs*ddlm}tdur|rtjatS)z5 Ensures aux_stream is initialized only once rr7N)r;r8r is_cuda_aliker,rrr7r'r'r( aux_stream,s  rr)maxsizecuda_visible_devicescCsvddl}ddl}ddlm}|jsdS|r't|jdr$|jnd}n|j }|dkr7|j }|S|}|S)Nrr7_device_count_amdsmir) torch.cuda torch.versionr;r8r _is_compiledrrr_device_count_nvml_C_cuda_getDeviceCount)rr,r8 raw_countrr'r'r(_cuda_device_count_stateless:s      rcCs ttjS)zGet number of CUDA devices, caching based on the value of CUDA_VISIBLE_DEVICES at the time of call. This should be used instead of torch.cuda.device_count() unless CUDA_VISIBLE_DEVICES has already been set to the desired value.)renvsr9r'r'r'r(cuda_device_count_statelessXs rcCs*t|tjr|dkrtjj|S|S)z Create a weak reference to a tensor. The new tensor will share the same data as the original tensor, but will not keep the original tensor alive. This ignores 0-size tensors as those don't allocate any memory. r)rsr,Tensornumelrmrweak_ref_tensor)rHr'r'r(resrtensorscCst|tjr t|St|trdd|DSt|tr$tdd|DSddlm}t||r=|dd|j D}|St d ) z Convenience function to create weak references to tensors, for single tensor, list of tensors or tuple of tensors. cSsg|]}t|qSr'rr^rr'r'r( sz$weak_ref_tensors..css|]}t|VqdSr]rrr'r'r(r`rz#weak_ref_tensors..rr cSsi|] \}}|t|qSr'r)r^rdvalr'r'r( sz$weak_ref_tensors..zInvalid type for tensors) rsr,rrrr vllm.sequencerritemsru)rrretr'r'r(weak_ref_tensorsrs     r  cpu_tensorcCs|sJdtjj|S)zQ Get a CUDA view of a CPU tensor using Unified Virtual Addressing (UVA). zCPU tensor must be pinned) is_pinnedr,rmrget_cuda_view_from_cpu_tensor)r r'r'r(rsr torch_versiontargetcCst|t|kSr])rparse)rrr'r'r(_is_torch_equal_or_newersrcCs>z tttj|WStyttjdt|kYSw)zCheck if the installed torch version is >= the target version. Args: target: a version string, like "2.6.0". Returns: Whether the condition meets. r,) rrAr, __version__r@r importlibmetadatarrr'r'r(is_torch_equal_or_newers   rcCsF|ddks Jttj}t|}|t|ko"t|d|kS)N.rKz.1)countrAr,rrr)rrr'r'r(_is_torch_equals   rcCs6zt|WStyttjdt|kYSw)zCheck if the installed torch version is == the target version. Args: target: a version string, like "2.6.0". Returns: Whether the condition meets. r,)rr@r rrrrr'r'r(is_torch_equals   rcCstdotjS)Nz 2.8.0.dev)rr, distributedis_xccl_availabler'r'r'r( supports_xcclsrriFRAGMENTr'op_nameop_func mutates_args fake_impl target_lib dispatch_keytags.c Csx|durg}|durddlm}|j}t||d}|pt} | j|||d| j|||d|dur:| ||dSdS)a `torch.library.custom_op` can have significant overhead because it needs to consider complicated dispatching logic. This function directly registers a custom op and dispatches it to the CUDA backend. See https://gist.github.com/youkaichao/ecbea9ec9fc79a45d2adce1784d7a9a5 for more details. By default, the custom op is registered to the vLLM library. If you want to register it to a different library, you can pass the library object to the `target_lib` argument. IMPORTANT: the lifetime of the operator is tied to the lifetime of the library object. If you want to bind the operator to a different library, make sure the library object is alive when the operator is used. Nrr7)r")r&)r%)r;r8r%r vllm_libdefineimpl_register_fake) r r!r"r#r$r%r&r8 schema_strmy_libr'r'r(direct_register_custom_ops  r-r])NNrr)NNr)NNNNr')n contextlibimportlib.metadatarr=r threadingcollections.abcrr functoolsrtypingrrrnumpyr numpy.typingnptr, packagingrpackaging.versionr torch.libraryr r vllm.envsr vllm.configr rrobjectlogging getLoggerrrrrrrruint8r float8_e4m3fnrtfloat64int32int64rrrrrLr)contextmanagerr*r0intr6rFrJrOr[rerorArwrrrrrrrrrrrr DTypeLikeNDArrayrrrrrlocalrrrrrr__annotations__rrrrr rrrrrrr'Tagr-r'r'r'r(s                5         /    0      .