o ۾ipX@sJddlZddlZddlZddlZddlZddlmZddlmZm Z m Z ddl Z ddl m Z ddlmZermddlmZmZddlmZddlmZmZdd lmZdd lmZmZdd lmZdd lm Z dd l!m"Z"ne#Z e e$Z%de&fddZ'Gdddej(Z)Gdddej(Z*Gddde Z+GdddZ,Gddde,Z-dS)N) timedelta) TYPE_CHECKINGAny NamedTuple) init_logger)AttentionBackendEnum) PrefixStore ProcessGroup) VllmConfig)ProcessorInputs PromptType) PoolingParams) DictPrompt TokPrompt)SamplingParams)FlexibleArgumentParser)AttentionSelectorConfigreturncCsddtvS)N microsoft )joinplatformunamelowerrrL/home/ubuntu/.local/lib/python3.10/site-packages/vllm/platforms/interface.pyin_wsl src@DeZdZeZeZeZeZeZ eZ eZ dS) PlatformEnumN) __name__ __module__ __qualname__enumautoCUDAROCMTPUXPUCPUOOT UNSPECIFIEDrrrrr% rc@r) CpuArchEnumN) rr r!r"r#X86ARMPOWERPCS390XRISCVOTHERUNKNOWNrrrrr,/r+r,c@seZdZUeed<eed<dedefddZdedefddZdedefd d Z dedefd d Z dedefd dZ de fddZ defddZdS)DeviceCapabilitymajorminorotherrcCs&t|tstS|j|jf|j|jfkSN isinstancer4NotImplementedr5r6selfr7rrr__lt__= zDeviceCapability.__lt__cCs&t|tstS|j|jf|j|jfkSr8r9r<rrr__le__Br?zDeviceCapability.__le__cCs&t|tstS|j|jf|j|jfkSr8r9r<rrr__eq__Gr?zDeviceCapability.__eq__cCs&t|tstS|j|jf|j|jfkSr8r9r<rrr__ge__Lr?zDeviceCapability.__ge__cCs&t|tstS|j|jf|j|jfkSr8r9r<rrr__gt__Qr?zDeviceCapability.__gt__cCs|jd|jS)N.)r5r6r=rrras_version_strVszDeviceCapability.as_version_strcCs.d|jkr dksJJ|jd|jS)z Express device capability as an integer ``. It is assumed that the minor version is always a single digit. r )r6r5rErrrto_intYszDeviceCapability.to_intN)rr r!int__annotations__rboolr>r@rArBrCstrrFrHrrrrr49s r4c @seZdZUeed<eed<eed<dZeed<dZeed<dZeed <gZ e eed <d Z eed <dZ eed <gZ e eed<gZe eed<dZedBed<edefddZede ejfddZdefddZdefddZdefddZdefddZdefdd Zdefd!d"Zdefd#d$Zd%edefd&d'Zdefd(d)Z defd*d+Z!e"defd,d-Z#e"defd.d/Z$e"d0efd1d2Z%e"dd3d4Z&e"d5d6d7d8defd9d:Z'e"de d6fd;d<Z(e" dd=ed>ejd?d@dd6fdAdBZ)e" Cdd0ede*dBfdDdEZ+e" CddFe,eefeBd0edefdGdHZ-e" CddFe,eefeBd0edefdIdJZ.e" CddFed0edefdKdLZ/e"dd0edefdMdNZ0e"dd0edefdOdPZ1e"dd0edefdQdRZ2e"dSdTZ3e"dUej4ddfdVdWZ5e" ddXe6dBddfdYdZZ7e"dd]d^Z8e"d_eddfd`daZ9e"dbeddfdcddZ:e"de;fdedfZj?dBde@fdidjZAe"defdkdlZBe"d>ejde,e@e@ffdmdnZCe"defdodpZDe"defdqdrZEe"defdsdtZFe"defdudvZGe"defdwdxZHe"defdydzZIe"dejfd{d|ZJe"defd}d~ZKe"defddZLe"defddZMe"  dddZNdefddZOdefddZPe"defddZQe"d?edddededeRddf ddZSe"d>ejfddZTe"defddZUe"defddZVe"defddZWe"ddZXe"deYee,edfffddZZe"dedBfddZ[e"dedefddZ\e"deYeeffddZ]dS)Platform_enum device_name device_typer( dispatch_keyray_device_key'VLLM_DEVICE_CONTROL_ENV_VAR_PLACEHOLDERdevice_control_env_varray_noset_device_env_varsinductorsimple_compile_backend dist_backendsupported_quantizationadditional_env_varsN_global_graph_poolrcCdS)z3Inductor config key for the PassManager custom passpost_grad_custom_post_passrrErrrpass_keyzPlatform.pass_keycCstjtjtjgS)z6Returns the supported dtypes for the current platform.)torchbfloat16float16float32rErrrsupported_dtypesszPlatform.supported_dtypescC |jtjkSr8)rNrr$rErrris_cuda zPlatform.is_cudacCrfr8)rNrr%rErrris_rocmrhzPlatform.is_rocmcCrfr8)rNrr&rErrris_tpurhzPlatform.is_tpucCrfr8)rNrr'rErrris_xpurhzPlatform.is_xpucCrfr8)rNrr(rErrris_cpurhzPlatform.is_cpucCrfr8)rNrr)rErrris_out_of_treerhzPlatform.is_out_of_treecCrfr8)rNrr*rErrris_unspecifiedrhzPlatform.is_unspecified prompt_lencCtjSr8)sysmaxsize)r=rorrrget_max_output_tokensszPlatform.get_max_output_tokenscC|jtjtjfvS)z1Stateless version of [torch.cuda.is_available][].rNrr$r%rErrr is_cuda_alikeszPlatform.is_cuda_alikecCrtr8rurErrris_sleep_mode_availablesz Platform.is_sleep_mode_availablecCr])z Get the pass manager class for this platform. It will be registered as a custom pass under the current_platform.pass_key. z8vllm.compilation.passes.pass_manager.PostGradPassManagerrclsrrrget_pass_manager_clszPlatform.get_pass_manager_clscCs|jS)zF Get the custom compile backend for current platform. )rXrxrrrget_compile_backendszPlatform.get_compile_backend device_idcCsB|jtjvrtj|jdkrtj|jd}||}t|S|S)NrR,)rUosenvironsplitrI)ryr} device_idsphysical_device_idrrrdevice_id_to_physical_device_ids z(Platform.device_id_to_physical_device_idc Csvzddl}Wnty}z td|WYd}~nd}~wwtt ddl}WddS1s4wYdS)z'Import any platform-specific C kernels.rNz!Failed to import from vllm._C: %r)vllm._C ImportErrorloggerwarning contextlibsuppress vllm._moe_C)ryvllmerrrimport_kernelss   "zPlatform.import_kernelsselected_backendrattn_selector_configrcCr])z,Get the attention backend class of a device.rRr)ryrrrrrget_attn_backend_clszPlatform.get_attn_backend_clscCstjgSr8)r TORCH_SDPArxrrrget_supported_vit_attn_backendssz(Platform.get_supported_vit_attn_backends head_sizedtypebackendzAttentionBackendEnum | NonecCs\|dur!||vsJd|d|td|d|StdtjdtjS)a Get the vision attention backend class of a device. NOTE: ViT Attention should be checked and override in the platform-specific implementation. we should not override this in any other places, like the model_executor/models/.py. We check if the backend is None or not: 1. If not, check if the backend is supported by the platform. 2. If None, continue to the default selection logic. NzBackend z; is not supported for vit attentionSupported backends are: zUsing backend z for vit attentionzUsing default backend )rr info_oncerr)ryrrrrrrget_vit_attn_backends zPlatform.get_vit_attn_backendrcCr])z:Stateless version of [torch.cuda.get_device_capability][].Nrryr}rrrget_device_capabilityr{zPlatform.get_device_capability capabilitycCs6|j|d}|dur dSt|tr||kS||kS)a. Test whether this platform is compatible with a device capability. The `capability` argument can either be: - A tuple `(major, minor)`. - An integer ``. (See [`DeviceCapability.to_int`][vllm.platforms.interface.DeviceCapability.to_int]) r}NFrr:tuplerHryrr}current_capabilityrrrhas_device_capability   zPlatform.has_device_capabilitycCs6|j|d}|dur dSt|tr||kS||kS)a3 Test whether this platform has exactly the specified device capability. The `capability` argument can either be: - A tuple `(major, minor)`. - An integer ``. (See [`DeviceCapability.to_int`][vllm.platforms.interface.DeviceCapability.to_int]) rNFrrrrris_device_capability3rzPlatform.is_device_capabilitycCs,|j|d}|dur dS|d|dkS)z Returns True if the device capability is any .x. Mirrors CUDA 13 'family' architecture semantics (e.g. 10.x, 11.x, 12.x). rNFrG)rrHrrrris_device_capability_familyKs z$Platform.is_device_capability_familycCt)zGet the name of a device.NotImplementedErrorrrrrget_device_nameZr`zPlatform.get_device_namecCr)z.Get the uuid of a device, e.g. the PCI bus ID.rrrrrget_device_uuid_r`zPlatform.get_device_uuidcCr)z*Get the total memory of a device in bytes.rrrrrget_device_total_memorydr`z Platform.get_device_total_memorycCs tjddS)aA device-specific wrapper of `torch.inference_mode`. This wrapper is recommended because some hardware backends such as TPU do not support `torch.inference_mode`. In such a case, they will fall back to `torch.no_grad` by overriding this method. T)mode)rainference_moderxrrrris zPlatform.inference_modedevicecCr)z: Set the device for the current platform. rryrrrr set_deviceszPlatform.set_deviceparsercCr])a Do some pre-registration or update action for the current platform. This function is called before global VllmConfig is initialized or cli arguments are parsed. It's used for out-of-tree platforms to register or update the configuration. For example, the out-of-tree quantization config can be imported and registered here dynamically. Nr)ryrrrrpre_register_and_updatezsz Platform.pre_register_and_update vllm_configr cCr])ac Check and update the configuration for the current platform. It can raise an exception if the configuration is not compatible with the current platform, or it can update the configuration to make it compatible with the current platform. The config is passed by reference, so it can be modified in place. Nr)ryrrrrcheck_and_update_configs z Platform.check_and_update_config model_archcCr])a  Verify whether the current platform supports the specified model architecture. - This will raise an Error or Warning based on the model support on the current platform. - By default all models are considered supported. Nr)ryrrrrverify_model_archs zPlatform.verify_model_archquantcCs.|jr||jvrt|d|jddSdS)zW Verify whether the quantization is supported by the current platform. z, quantization is currently not supported in rDN)rZ ValueErrorrO)ryrrrrverify_quantizations zPlatform.verify_quantizationcCsrt}|dvr tjS|ds|drtjS|dr"tjS|dkr)tjS|dr1tj S|r6tj Stj S)z Determine the CPU architecture of the current system. Returns CpuArchEnum indicating the architecture type. )x86_64amd64i386i686armaarchppcs390xriscv) rmachinerr,r- startswithr.r/r0r1r2r3)ryrrrrget_cpu_architectures   zPlatform.get_cpu_architecturecCstr tddSdS)z?Checks whether pin memory is available on the current platform.zPUsing 'pin_memory=False' as WSL is detected. This may slow down the performance.FT)rrrrxrrris_pin_memory_availables z Platform.is_pin_memory_availablecCr)z3 Return the memory usage in bytes. rrrrrget_current_memory_usagerz!Platform.get_current_memory_usagecCr)zA Return the punica wrapper for current platform. rrxrrrget_punica_wrapperrzPlatform.get_punica_wrappercCstdtdfS)zE Return the platform specific values for (-inf, inf) z-infinf)floatryrrrrget_infinity_valuesszPlatform.get_infinity_valuescCr])zF Checks if the platform allows inplace memory updates Trrxrrrcan_update_inplacerzPlatform.can_update_inplacecCr])zK Returns how much padding the LoRA logits need for kernels rrxrrrget_lora_vocab_padding_sizerz$Platform.get_lora_vocab_padding_sizecCr])zW Get device specific communicator class for distributed communication. zUvllm.distributed.device_communicators.base_device_communicator.DeviceCommunicatorBaserrxrrrget_device_communicator_clsrz$Platform.get_device_communicator_clscCr])zI Returns whether the current platform supports MX types. Frrxrrr supports_mxrzPlatform.supports_mxcCr])zJ Returns whether the current platform supports FP8 types. Frrxrrr supports_fp8rzPlatform.supports_fp8cCr])a Returns whether the preferred FP8 type is FNUZ on the current platform. There are two representations of FP8, OCP FP8 and FNUZ FP8. The OCP specification can be found at https://tinyurl.com/b7jvwpft. The FNUZ specification can be found at https://tinyurl.com/5n6hwwu5. AMD's MI300 and MI325 have native hardware support for FNUZ. All other hardware has converged on the OCP FP8 standard. Frrxrrr is_fp8_fnuz  zPlatform.is_fp8_fnuzcCrp)z Returns the preferred FP8 type on the current platform. See the documentation for is_fp8_fnuz for details. )ra float8_e4m3fnrxrrr fp8_dtypeszPlatform.fp8_dtypecCr])zS Whether to use allgather in LogitsProcessor to gather the logits. Trrxrrruse_all_gather rzPlatform.use_all_gathercCr])zR Returns if custom allreduce is supported on the current platform Frrxrrruse_custom_allreduce'rzPlatform.use_custom_allreducecCr])zu Returns True if we register attention as one giant opaque custom op on the current platform Frrxrrropaque_attention_op.r{zPlatform.opaque_attention_opprompt#PromptType | DictPrompt | TokPromptparamsSamplingParams | PoolingParamsprocessed_inputsr cCr])z6Raises if this request is unsupported on this platformNr)ryrrrrrrvalidate_request6szPlatform.validate_requestkeycCsJtt|jd}|durt||rt||}|dur|Std|j|dS)Nz1Current platform %s does not have '%s' attribute.)getattrrarPhasattrrr)r=rrattrrrr __getattr__?s zPlatform.__getattr__cCs |j}|jdur ||_|jS)zA Return the global graph pool for this platform. N) __class__r\graph_pool_handle)r=ryrrrget_global_graph_poolOs  zPlatform.get_global_graph_poolcCr])zB Get static graph wrapper class for static graph. z=vllm.compilation.base_static_graph.AbstractStaticGraphWrapperrrxrrrget_static_graph_wrapper_clsXrz%Platform.get_static_graph_wrapper_cls prefix_storer group_rank group_sizetimeoutr cCr)zI Init platform-specific torch distributed process group. r)ryrrrrrrrr#stateless_init_device_torch_dist_pg_rz,Platform.stateless_init_device_torch_dist_pgcCr)zJ Check if the dtype is supported by the current platform. rrrrrcheck_if_supports_dtypemrz Platform.check_if_supports_dtypecCr])zV Returns if the hybrid kv cache is supported by the current platform. Frrxrrrsupport_hybrid_kv_cachetrz Platform.support_hybrid_kv_cachecCr])zQ Returns if the graph mode is supported by the current platform. Frrxrrrsupport_static_graph_mode{rz"Platform.support_static_graph_modecCr])zN Returns if the current platform needs to sync weight loader. Frrxrrruse_sync_weight_loaderrzPlatform.use_sync_weight_loadercs|sSfdd}|S)zD Wrap the original weight loader to make it synced. cs4|g|Ri|}|jtdkrt||S)Ncpu)rra_sync)paramargskwargsoutoriginal_weight_loaderrr_synced_weight_loaders zAPlatform.make_synced_weight_loader.._synced_weight_loader)r)ryrrrrrmake_synced_weight_loaders z"Platform.make_synced_weight_loader.cCiS)zo Returns a mapping from device_type to a tuple of supported kv_buffer_device for nixl. rrxrrrget_nixl_supported_devicesr{z#Platform.get_nixl_supported_devicescCr])zH Returns the nixl memory type for the current platform. Nrrxrrrget_nixl_memory_typerzPlatform.get_nixl_memory_type max_model_lencCs|S)z? Check max_model_len for the current platform. r)ryrrrrcheck_max_model_lenrzPlatform.check_max_model_lencOr)zX Set some additional forward context for the current platform if needs. r)ryrrrrrset_additional_forward_contextrz'Platform.set_additional_forward_context)rNr8)r)rr rN)rrrrrr rN)^rr r!rrJrLrQrSrUrVlistrXrYrZr[r\rpropertyr_rarrerKrgrirjrkrlrmrnrIrsrvrw classmethodrzr|rrrrrr4rrrrrrrrrrrrrrrrr,rrtypesDevicerrrrrrrrrrrrrrrrrrrrrrrrrdictrrrrrrrrrMcs                               rMc@seZdZejZdZdS)UnspecifiedPlatformrRN)rr r!rr*rNrPrrrrr sr ).rr"rrrqdatetimertypingrrrra vllm.loggerr#vllm.v1.attention.backends.registryrtorch.distributedrr vllm.configr vllm.inputsr r vllm.pooling_paramsr vllm.renderers.inputsrrvllm.sampling_paramsrvllm.utils.argparse_utilsrvllm.v1.attention.selectorrobjectrrrKrEnumrr,r4rMr rrrrs>         *X