o U۷i$@s`ddlZddlZddlZddlZddlZddlZddZer!dndZejee dZ gdZ e D] Z e e e ee <q0dd lmZmZd Zejd d dejd d fddZejfddZee j_ee_ddZ    d#ddZ    d#ddZee_ee_ ddZ!ddZ"ere!ne"ddl#m#Z#m$Z$m%Z%ddl&m'Z'ddl(m)Z)de*d e)fd!d"Z+dS)$NcCs tjdS)Nwin)sysplatform startswithrrD/home/ubuntu/vllm_env/lib/python3.10/site-packages/cudnn/__init__.py is_windows s rz.Release._compiled_modulez._compiled_module)package)backend_versionbackend_version_stringget_last_error_stringdestroy_handlenorm_forward_phasereduction_mode behavior_note knob_type create_handlecreate_kernel_cachecreate_device_properties get_streamnumerical_note set_streambuild_plan_policy data_typetensor_reordering heur_modepygraphtensorknobcudnnGraphNotSupportedErrordiagonal_alignmentattention_implementation) _library_type_is_torch_tensorz1.18.0Fc Cs |j||t||||||| d S)ag Create a tensor. Args: dim (List[int]): The dimensions of the tensor. stride (List[int]): The strides of the tensor. data_type (cudnn.data_type): The data type of the tensor. is_virtual (bool): Flag indicating if the tensor is virtual. is_pass_by_value (bool): Flag indicating if the tensor is passed by value. ragged_offset (cudnn_tensor): The ragged offset tensor. reordering_type (cudnn.tensor_reordering): The reordering type of the tensor. name (str): The name of the tensor. Returns: cudnn_tensor: The created tensor. ) dimstrider is_virtualis_pass_by_value ragged_offsetreordering_typenameuid) _make_tensorr#) selfr'r(rr)r*r+r,r-r.rrr_tensor4sr1cCs|t|SN)_set_data_typer#)r0rrrrr3]sr3cCs*t|tur|St|r|St|Sr2)typeintr$data_ptr_pybind_module _get_data_ptr) input_tensorrrr_library_device_pointerhs  r:c Cs,dd|D}t|}||||dS)aQ Execute a cudnn graph. Args: tensor_to_device_buffer (dict(cudnn_tensor, Union[torch.Tensor, int, __dlpack__])): The dimensions of the tensor. workspace (Union[torch.Tensor, int, __dlpack__]): The name of the tensor. handle: cudnn_handle created with cudnn.create_handle() Returns: None cS6i|]\}}|durt|tur|n|t|qSr2r4r5get_uidr:.0xpointerrrr 0z_execute..N)itemsr:_execute) r0tensor_to_device_buffer workspacehandle override_uidsoverride_shapesoverride_stridesuid_to_tensor_pointerworkspace_pointerrrrrEts rEc Cs4dd|D}t|} ||| |||||dS)a Execute a cudnn graph. Args: tensor_to_device_buffer (dict(cudnn_tensor, Union[torch.Tensor, int, __dlpack__])): The dimensions of the tensor. workspace (Union[torch.Tensor, int, __dlpack__]): The name of the tensor. index(int): Location of execution plan to use. handle: cudnn_handle created with cudnn.create_handle() Returns: None cSr;r2r<r>rrrrBrCz*_execute_plan_at_index..N)rDr:_execute_plan_at_index) r0rFrGindexrHrIrJrKrLrMrrrrNsrNcCs|ttjtdd}|r(t|dksJdt|dtj |d}ntj d}t |j tj j }t|dS)Npurelibznvidia/cudnn/bin/cudnn64_9.dllr"Found z% libcudnn.dll.x in nvidia-cudnn-cuXX.rz cudnn64_9.dll)globospathjoin sysconfigget_pathlenctypeswindll LoadLibrarycast_handlec_void_pvaluer7_set_dlhandle_cudnnlib_pathlibrHrrr load_cudnns  rdc Csttjtdd}|sttjtdd}|r6t|dks.Jdt|dt|d}n$ztd}Wnt yYztd }Wn t yVd}YnwYnw|durnt |j tj j }t|dSdS) NrPz#nvidia/cudnn/lib/libcudnn.so.*[0-9]z'nvidia/cudnn_jit/lib/libcudnn.so.*[0-9]r"rQz$ libcudnn.so.x in nvidia-cudnn-cuXX.rz libcudnn.so.9z libcudnn.so)rRrSrTrUrVrWrXrYCDLL Exceptionr\r]r^r_r7r`rarrr _dlopen_cudnns(   rg)graphjit graph_cache)Graph)Anyr-returnc Cs,|dkr"z ddlm}|WSty!}ztd||d}~ww|dkrDz ddlm}|WStyC}ztd||d}~ww|dkrfz dd lm}|WStye}ztd ||d}~ww|d krz dd lm}|WSty}ztd ||d}~ww|dkrz ddlm }|WSty}ztd||d}~ww|dkrz ddl m }|WSty}ztd||d}~ww|dkrz ddl m }|WSty}ztd||d}~ww|dkrz ddl m } | WSty}ztd||d}~wwt|)NNSAr")rnz_NSA requires optional dependencies. Install with 'pip install nvidia-cudnn-frontend[cutedsl]': GemmSwigluSm100)rozkGemmSwigluSm100 requires optional dependencies. Install with 'pip install nvidia-cudnn-frontend[cutedsl]': gemm_swiglu_wrapper_sm100)rpzugemm_swiglu_wrapper_sm100 requires optional dependencies. Install with 'pip install nvidia-cudnn-frontend[cutedsl]': GemmAmaxSm100)rqziGemmAmaxSm100 requires optional dependencies. Install with 'pip install nvidia-cudnn-frontend[cutedsl]': gemm_amax_wrapper_sm100)rrzsgemm_amax_wrapper_sm100 requires optional dependencies. Install with 'pip install nvidia-cudnn-frontend[cutedsl]': grouped_gemm)rszhgrouped_gemm requires optional dependencies. Install with 'pip install nvidia-cudnn-frontend[cutedsl]': GroupedGemmSwigluSm100)rtzrGroupedGemmSwigluSm100 requires optional dependencies. Install with 'pip install nvidia-cudnn-frontend[cutedsl]': !grouped_gemm_swiglu_wrapper_sm100)ruz}grouped_gemm_swiglu_wrapper_sm100 requires optional dependencies. Install with 'pip install nvidia-cudnn-frontend[cutedsl]': )native_sparse_attentionrnrf ImportError gemm_swiglurorp gemm_amaxrqrrr%rsrtruAttributeError) r-_NSAe_GemmSwigluSm100_gemm_swiglu_wrapper_sm100_GemmAmaxSm100_gemm_amax_wrapper_sm100 _grouped_gemm_GroupedGemmSwigluSm100"_grouped_gemm_swiglu_wrapper_sm100rrr __getattr__s         r)NNNN),rYrRrSrrV importlibr module_name import_module__name__r7symbols_to_import symbol_namegetattrglobals datatypesr#r$ __version__rNOT_SETrNONEr1r3r set_data_typerr:rErNexecuteexecute_plan_at_indexrdrgrhrirjwrapperrktypingrlstrrrrrrs` +    %