o پiO @sRUddlmZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z m Z ddlZer3ddlmZe dedefdZd7d d Zd8ddZed9ddZeZeedgZddgZgdZgZe eeeejfZde d<Gddde!eZ"ejdej#dej$diZ%d:d d!Z&dddddddddd" d;d0d1Z'ed) annotationsN) TYPE_CHECKINGAnyCallableListTuple TypeAliasTypeVarUnion)ModuleF.)boundfnreturncs itfdd}|S)z NOTE: `functools.lru_cache` is not compatible with `torch.compile` So we manually implement a simple cache_once decorator to replace it. cs>|tt|dddf}|vr|i||<|S)NcSs|dS)Nr)xrrK/home/ubuntu/.local/lib/python3.10/site-packages/sglang/jit_kernel/utils.pysz-cache_once..wrapper..)key)tuplesorteditems)argskwargsrr result_maprrwrapperszcache_once..wrapper) functoolswraps)rrrrr cache_oncesrtupTuple[str, str]strcCs|\}}d|d|dS)NzTVM_FFI_DLL_EXPORT_TYPED_FUNC(z, (z));r)r export_name kernel_namerrr _make_wrapper"sr% pathlib.PathcsDttjfdd}dd}|p|}|dur td|S)Ncs(}|dr|dr|SdS)Nincludecsrc)resolveexists) candidatecur_dirrr_environment_install,sz2_resolve_kernel_path.._environment_installcSsdSNrrrrr_package_install2sz._resolve_kernel_path.._package_installzCannot find sgl-kernel/jit path)pathlibPath__file__parentr) RuntimeError)r.r0pathrr,r_resolve_kernel_path's  r7r' -std=c++20-O3)r8r9z--expt-relaxed-constexprrCPP_TEMPLATE_TYPEc@seZdZdddZdS) CPPArgListrr"cCs d|S)Nz, )join)selfrrr__str__Es zCPPArgList.__str__Nrr")__name__ __module__ __qualname__r>rrrrr;Dsr;fp32_tfp16_tbf16_trcs d ddtfdd|DS) Nargr:rr"cSsRt|tr |r dSdSt|ttfrt|St|tjr t|Stdt |)Ntruefalsez,Unsupported argument type for cpp template: ) isinstanceboolintfloatr"torchdtype CPP_DTYPE_MAP TypeErrortype)rFrrr_convertQs   zmake_cpp_args.._convertc3s|]}|VqdSr/r.0rFrRrr Zz make_cpp_args..)rFr:rr")r;)rrrUr make_cpp_argsPs  rX) cpp_files cuda_files cpp_wrappers cuda_wrappers extra_cflagsextra_cuda_cflags extra_ldflagsextra_include_pathsbuild_directoryrYList[str] | NonerZr[List[Tuple[str, str]] | Noner\r]r^r_r`ra str | Noner c  Gs"ddlm} |p g}|p g}|pg}|pg}|pg}|pg}|p!g}|p%g}dd|D} dd| D} | dd|D7} dd|D} dd| D}|d d|D7}d }|tjv}|scttj|<z&| d d d d| D| |t|t|t|t ||dW|stj|=SS|stj|=w)a{ Loading a JIT module from C++/CUDA source files. We define a wrapper as a tuple of (export_name, kernel_name), where `export_name` is the name used to called from Python, and `kernel_name` is the name of the kernel class in C++/CUDA source. :param args: Unique marker of the JIT module. Must be distinct for different kernels. :type args: str :param cpp_files: A list of C++ source files. :type cpp_files: List[str] | None :param cuda_files: A list of CUDA source files. :type cuda_files: List[str] | None :param cpp_wrappers: A list of C++ wrappers, defining the export name and kernel name. :type cpp_wrappers: List[Tuple[str, str]] | None :param cuda_wrappers: A list of CUDA wrappers, defining the export name and kernel name. :type cuda_wrappers: List[Tuple[str, str]] | None :param extra_cflags: Extra C++ compiler flags. :type extra_cflags: List[str] | None :param extra_cuda_cflags: Extra CUDA compiler flags. :type extra_cuda_cflags: List[str] | None :param extra_ldflags: Extra linker flags. :type extra_ldflags: List[str] | None :param extra_include_paths: Extra include paths. :type extra_include_paths: List[str] | None :param build_directory: The build directory for JIT compilation. :type build_directory: str | None :return: A just-in-time(JIT) compiled module. :rtype: Module r) load_inlinecSg|] }td|qSr( KERNEL_PATHr)rTfrrr zload_jit..cSg|]}d|dqSz #include ""rrTr6rrrrlcSg|]}t|qSrr%rTr rrrrlcSrfrgrhrjrrrrlrmcSrnrorrqrrrrlrrcSrsrrtrurrrrlrvTVM_FFI_CUDA_ARCH_LISTsgl_kernel_jit__css|]}t|VqdSr/)r"rSrrrrVrWzload_jit..) cpp_sources cuda_sourcesr]r^r_r`ra) tvm_ffi.cppreosenviron_get_cuda_arch_listr<DEFAULT_CFLAGSDEFAULT_CUDA_CFLAGSDEFAULT_LDFLAGSDEFAULT_INCLUDE)rYrZr[r\r]r^r_r`rarre cpp_pathsrz cuda_pathsr{env_key env_existedrrrload_jit]sD *    rrJcCs&ddl}|j}|j|ddkS)Nr rMcudacurrent_deviceget_device_capability)rMdevicerrris_arch_support_pdls rcCs(tj}tj|\}}|d|S)zDGet the correct CUDA architecture string for TVM_FFI_CUDA_ARCH_LIST..r)rmajorminorrrrrs r)rr rr )r r!rr")rr&)rr:rr;)rr"rYrbrZrbr[rcr\rcr]rbr^rbr_rbr`rbrardrr )rrJr?)* __future__rrr}r1typingrrrrrrr r rMtvm_ffir r rr%r7rir"rrrrrKrLrJrNr:__annotations__listr;float16bfloat16rOrXrrrrrrrsN(     U