o Gۂid@szddlZddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z ddl m Z ddlmZejejeZejedgZejed Zd gZdaed d Zed dZGdddeZddZddddddZddddddZ dZ!e"e!Z#ddZ$e%dde&d DZ'd!e'd"<d"e'd#<d#e'd!<d$d%Z(d&d'Z)Gd(d)d)eZ*Gd*d+d+eZ+dS),N)Path)knobs)compile_module_from_src) _allocation) GPUTarget) GPUDriverincludelib libcuda.so.1cCstjj}r |gStddgjdd}dd|D}dd|D}td}|r7|s7d d| d D}d }|rH|d t |7}|d 7}n|d7}|d7}t dd|Ds]J||S)Nz/sbin/ldconfigz-pignore)errorscSs g|] }d|vr|dqS)r )split).0liner`/home/ubuntu/maya3_transcribe/venv/lib/python3.10/site-packages/triton/backends/nvidia/driver.py s z libcuda_dirs..cSsg|]}tj|qSr)ospathdirname)rlocrrrrsLD_LIBRARY_PATHcSs&g|]}tjtj|dr|qS)r rrexistsjoin)rdirrrrr s&:zlibcuda.so cannot found! z!Possible files are located at %s.z:Please create a symlink of libcuda.so to any of the files.z(s$zlibcuda_dirs..) rnvidia libcuda_path subprocess check_outputdecode splitlinesrgetenvrstrany)env_libcuda_pathlibslocsdirsenv_ld_library_pathmsgrrr libcuda_dirss   r.cCs tgtSN) libdevice_dirr.rrrr library_dirs,s r1cs$eZdZfddZddZZS) CudaUtilscs"t|dstt|||_|jS)Ninstance)hasattrsuperr2__new__r3)cls __class__rrr68s zCudaUtils.__new__cCsXtttjtddttt d}|j a |j |_ |j |_ |j |_ |j|_|j|_dS)Nzdriver.c cuda_utilssrcnamer1 include_dirs libraries)rrrrrr read_textr1r>r? PyCUtensorMap load_binaryget_device_propertiescuOccupancyMaxActiveClustersset_printf_fifo_sizefill_tma_descriptor)selfmodrrr__init__=s zCudaUtils.__init__)__name__ __module__ __qualname__r6rI __classcell__rrr8rr26s r2cCs|ddkrdS|drdSiddddd d d d d ddddddddddddddddddddddd|S)Nr* CUdeviceptr tensordesc CUtensorMapi1int8_ti8i16int16_ti32int32_ti64int64_tu1uint8_tu8u16uint16_tu32uint32_tu64uint64_tfp16doublebf16fp32f32fp64 nvTmaDesc) startswith)tyrrr ty_to_cppSsN       rmr_rarc)rdrfrgrhri pack_fp16 pack_bf16 pack_fp32 pack_fp64 iiiKKppOOOOOOcsfdd}fddfddfdd||}d d t|D}d fd d |D}t|}g}|D]}||q?dd t|D}t|dkrddddd|Dnd } g} |D]&\} } | dkruql| tvr| t| d| ql| t| d| qld| } g}|D];\} } | ddkr|d| dq| tvr|d| dq| dkr|d| q| dkr|d| qt t|}d}dd |D}dd |D}dd |D}d d |D}|d!|d"d#t| dkrd| nd d$d|d%|fd&d |Dd'|d(| d)||d||d||d*t|dkrZdd|nd d+}|S),Nc sg}d}|D]q}t|trr|drrr|nd}|d7}td|}|d}|d}|dd}|durT|d|td|D]} |dqF|d n|d t|D]} |d q]t|D]} |dqiq||qr|t ksJ|S) NrrPztensordesc<([^[>]*)\[([^]]*)\],rNrYrRrjrW) isinstancer&rkrematchgroupcountappendrangelen) signatureoutputtensordesc_idxsigmetarxdtypeshapendim_)tensordesc_metarr_expand_signatures0           z(make_launcher.._expand_signaturecs0t|tr|D]}||qdS||dSr/)rvtupler{)rrx)_flatten_signaturerrrs  z)make_launcher.._flatten_signaturecsJt|trdt|}d|dS|ddkrdS|dvr!dSt|S)Nru[]rrNz PyObject* constexprrj)rvrrmaprmrlval_extracted_typerrrs   z&make_launcher.._extracted_typec srt|trdt|}d|dS|ddkrdS|dvr!dS|dr(dSd d d d d dddddd t|S)N()rrNOrrPdlbhiLBHIK) relongrSrVrXrZr\r_rarc)rvrrrrkrmr format_ofrrrs,     z make_launcher..format_ofcSi|]\}}||qSrrrrsrrr z!make_launcher..rcsg|]}|qSrr)rrlrrrrsz!make_launcher..cSrrrrrrrrrr, css|] \}}d|VqdS)z&_argNrrrrlrrrrsz make_launcher..rz argrNptr_infoz.dev_ptr_arg_storagerjz*tma_ptrz c Ss:g|]\}}|ddkrd|d|d|d|d qS)rrNzDevicePtrInfo ptr_infoz = getPointer(_argrz); if (!ptr_infoz.valid) return NULL;rrrrrrs  c Ss0g|]\}}|dkrd|d|d|dqS)rjzCUtensorMap* tma_ptrz = getTmaDesc(_argz); if (!tma_ptrz) return NULL;rrrrrrs c Ss<g|]\}}|tvrt|d|dt|d|dqS) _argz _storage = z(_argz);)FLOAT_STORAGE_TYPEFLOAT_PACK_FUNCTIONrrrrrs "cSs"g|] \}}|dkrd|qS)rz&argrrrrrrs"z&global_scratchz&profile_scratcha #include "cuda.h" #include #include #include #define PY_SSIZE_T_CLEAN #include typedef struct { PyObject_HEAD; _Alignas(128) CUtensorMap tensorMap; } PyCUtensorMapObject; static inline void gpuAssert(CUresult code, const char *file, int line) { if (code != CUDA_SUCCESS) { const char* prefix = "Triton Error [CUDA]: "; const char* str; cuGetErrorString(code, &str); char err[1024] = {0}; strcat(err, prefix); strcat(err, str); PyGILState_STATE gil_state; gil_state = PyGILState_Ensure(); PyErr_SetString(PyExc_RuntimeError, err); PyGILState_Release(gil_state); } } #define CUDA_CHECK(ans) { gpuAssert((ans), __FILE__, __LINE__); } typedef CUresult (*cuLaunchKernelEx_t)(const CUlaunchConfig* config, CUfunction f, void** kernelParams, void** extra); static cuLaunchKernelEx_t getLaunchKernelExHandle() { // Open the shared library void* handle = dlopen("libcuda.so.1", RTLD_LAZY); if (!handle) { PyErr_SetString(PyExc_RuntimeError, "Failed to open libcuda.so.1"); return NULL; } // Clear any existing error dlerror(); cuLaunchKernelEx_t cuLaunchKernelExHandle = (cuLaunchKernelEx_t)dlsym(handle, "cuLaunchKernelEx"); // Check for errors const char *dlsym_error = dlerror(); if (dlsym_error) { PyErr_SetString(PyExc_RuntimeError, "Failed to retrieve cuLaunchKernelEx from libcuda.so.1"); return NULL; } return cuLaunchKernelExHandle; } static void _launch(int gridX, int gridY, int gridZ, int num_warps, int num_ctas, int launch_cooperative_grid, int launch_pdl, int shared_memory, CUstream stream, CUfunction function, CUdeviceptr global_scratch, CUdeviceptr profile_scratchz) { void *params[] = { au }; if (gridX*gridY*gridZ > 0) { // 4 attributes that we can currently pass maximum CUlaunchAttribute launchAttr[4]; static cuLaunchKernelEx_t cuLaunchKernelExHandle = NULL; if (cuLaunchKernelExHandle == NULL) { cuLaunchKernelExHandle = getLaunchKernelExHandle(); } CUlaunchConfig config; config.gridDimX = gridX * num_ctas; config.gridDimY = gridY; config.gridDimZ = gridZ; config.blockDimX = 32 * num_warps; config.blockDimY = 1; config.blockDimZ = 1; config.sharedMemBytes = shared_memory; config.hStream = stream; config.attrs = launchAttr; int num_attrs = 0; if (launch_pdl != 0) { CUlaunchAttribute pdlAttr = { .id = CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION, .value = 1}; launchAttr[num_attrs] = pdlAttr; ++num_attrs; } if (launch_cooperative_grid != 0) { CUlaunchAttribute coopAttr = { .id = CU_LAUNCH_ATTRIBUTE_COOPERATIVE, .value = 1}; launchAttr[num_attrs] = coopAttr; ++num_attrs; } if (num_ctas != 1) { CUlaunchAttribute clusterAttr = {}; clusterAttr.id = CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION; clusterAttr.value.clusterDim.x = num_ctas; clusterAttr.value.clusterDim.y = 1; clusterAttr.value.clusterDim.z = 1; launchAttr[num_attrs] = clusterAttr; ++num_attrs; CUlaunchAttribute clusterSchedulingAttr = {}; clusterSchedulingAttr.id = CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE; clusterSchedulingAttr.value.clusterSchedulingPolicyPreference = CU_CLUSTER_SCHEDULING_POLICY_SPREAD; launchAttr[num_attrs] = clusterSchedulingAttr; ++num_attrs; } // num_ctas == 16 is non-portable. Does work for H100 and B200 tho config.numAttrs = num_attrs; if (num_ctas == 16) { CUDA_CHECK(cuFuncSetAttribute( function, CU_FUNC_ATTRIBUTE_NON_PORTABLE_CLUSTER_SIZE_ALLOWED, 1 )); } CUDA_CHECK(cuLaunchKernelExHandle(&config, function, params, 0)); } } typedef struct _DevicePtrInfo { CUdeviceptr dev_ptr; bool valid; } DevicePtrInfo; static PyObject* data_ptr_str = NULL; static PyObject* py_tensor_map_type = NULL; static inline DevicePtrInfo getPointer(PyObject *obj, int idx) { DevicePtrInfo ptr_info; ptr_info.dev_ptr = 0; ptr_info.valid = true; if (PyLong_Check(obj)) { ptr_info.dev_ptr = PyLong_AsUnsignedLongLong(obj); return ptr_info; } if (obj == Py_None) { // valid nullptr return ptr_info; } PyObject *ret = PyObject_CallMethodNoArgs(obj, data_ptr_str); if (!ret) { PyErr_SetString(PyExc_TypeError, "Pointer argument must be either uint64 or have data_ptr method"); ptr_info.valid = false; goto cleanup; } if (!PyLong_Check(ret)) { PyErr_SetString(PyExc_TypeError, "data_ptr method of Pointer object must return 64-bit int"); ptr_info.valid = false; goto cleanup; } ptr_info.dev_ptr = PyLong_AsUnsignedLongLong(ret); if(!ptr_info.dev_ptr) return ptr_info; uint64_t dev_ptr; int status = cuPointerGetAttribute(&dev_ptr, CU_POINTER_ATTRIBUTE_DEVICE_POINTER, ptr_info.dev_ptr); if (status == CUDA_ERROR_INVALID_VALUE) { PyErr_Format(PyExc_ValueError, "Pointer argument (at %d) cannot be accessed from Triton (cpu tensor?)", idx); ptr_info.valid = false; } else if (status != CUDA_SUCCESS) { CUDA_CHECK(status); // Catch any other cuda API errors ptr_info.valid = false; } ptr_info.dev_ptr = dev_ptr; cleanup: Py_XDECREF(ret); return ptr_info; } static inline CUtensorMap* getTmaDesc(PyObject *obj) { if (sizeof(CUtensorMap*) != 8) { PyErr_SetString(PyExc_SystemError, "getTmaDesc() requires 64-bit compilation"); return NULL; } if (Py_TYPE(obj) != (PyTypeObject*)py_tensor_map_type) { PyErr_Format(PyExc_TypeError, "object must be of type PyCUtensorMap, got %s", Py_TYPE(obj)->tp_name); return NULL; } CUtensorMap* map = &((PyCUtensorMapObject*)obj)->tensorMap; uintptr_t align_128 = (uintptr_t)map & (128 - 1); if (align_128 != 0) { PyErr_Format(PyExc_ValueError, "CUtensorMap must be aligned to 128B, but got (&map) mod 128 = %ld", align_128); return NULL; } return map; } static void ensureCudaContext() { CUcontext pctx; CUDA_CHECK(cuCtxGetCurrent(&pctx)); if (!pctx) { // Ensure device context. CUdevice device; CUDA_CHECK(cuDeviceGet(&device, 0)); CUDA_CHECK(cuDevicePrimaryCtxRetain(&pctx, device)); CUDA_CHECK(cuCtxSetCurrent(pctx)); } } static uint16_t pack_fp16(double f) { uint16_t result; // from https://github.com/python/pythoncapi-compat #if 0x030600B1 <= PY_VERSION_HEX && PY_VERSION_HEX <= 0x030B00A1 && !defined(PYPY_VERSION) _PyFloat_Pack2(f, (unsigned char*)&result, 1); #else PyFloat_Pack2(f, (unsigned char*)&result, 1); #endif return result; } static uint16_t pack_bf16(double f) { float f32 = (float)f; uint32_t u32 = *(uint32_t*)&f32; return (uint16_t)(u32 >> 16); } static uint32_t pack_fp32(double f) { float f32 = (float)f; return *(uint32_t*)&f32; } static uint64_t pack_fp64(double f) { return *(uint64_t*)&f; } static PyObject* launch(PyObject* self, PyObject* args) { // ensure cuda context is valid before calling any CUDA APIs, e.g. before getPointer calls cuPointerGetAttributes ensureCudaContext(); int gridX, gridY, gridZ; uint64_t _stream; uint64_t _function; int launch_cooperative_grid; int launch_pdl; PyObject *launch_enter_hook = NULL; PyObject *launch_exit_hook = NULL; PyObject *kernel_metadata = NULL; PyObject *launch_metadata = NULL; PyObject *global_scratch_obj = NULL; PyObject *profile_scratch_obj = NULL; cs$g|]\}}|d|dqS)r;rrrrrrs$z if(!PyArg_ParseTuple(args, "aM", &gridX, &gridY, &gridZ, &_stream, &_function, &launch_cooperative_grid, &launch_pdl, &global_scratch_obj, &profile_scratch_obj, &kernel_metadata, &launch_metadata, &launch_enter_hook, &launch_exit_hooka)) { return NULL; } int num_warps, num_ctas, shared_memory; if (!PyArg_ParseTuple(kernel_metadata, "iii", &num_warps, &num_ctas, &shared_memory)) { PyErr_SetString(PyExc_TypeError, "kernel_metadata must be a tuple"); return NULL; } // extract launch metadata if (launch_enter_hook != Py_None){ PyObject* ret = PyObject_CallOneArg(launch_enter_hook, launch_metadata); if (!ret) return NULL; Py_DECREF(ret); } CUdeviceptr global_scratch = 0; if (global_scratch_obj != Py_None) { DevicePtrInfo global_scratch_info = getPointer(global_scratch_obj, -1); if (!global_scratch_info.valid) { return NULL; } global_scratch = global_scratch_info.dev_ptr; } CUdeviceptr profile_scratch = 0; if (profile_scratch_obj != Py_None) { DevicePtrInfo profile_scratch_info = getPointer(profile_scratch_obj, -1); if (!profile_scratch_info.valid) { return NULL; } profile_scratch = profile_scratch_info.dev_ptr; } // raise exception asap z Py_BEGIN_ALLOW_THREADS; _launch(gridX, gridY, gridZ, num_warps, num_ctas, launch_cooperative_grid, launch_pdl, shared_memory, (CUstream)_stream, (CUfunction)_function, global_scratch, profile_scratchap); Py_END_ALLOW_THREADS; if (PyErr_Occurred()) { return NULL; } if(launch_exit_hook != Py_None){ PyObject* ret = PyObject_CallOneArg(launch_exit_hook, launch_metadata); if (!ret) return NULL; Py_DECREF(ret); } Py_RETURN_NONE; } static PyMethodDef ModuleMethods[] = { {"launch", launch, METH_VARARGS, "Entry point for all kernels with this signature"}, {NULL, NULL, 0, NULL} // sentinel }; static struct PyModuleDef ModuleDef = { PyModuleDef_HEAD_INIT, "__triton_launcher", NULL, //documentation -1, //size ModuleMethods }; PyMODINIT_FUNC PyInit___triton_launcher(void) { data_ptr_str = PyUnicode_InternFromString("data_ptr"); if(data_ptr_str == NULL) { return NULL; } PyObject* driver_mod = PyImport_ImportModule("triton.backends.nvidia.driver"); if (driver_mod == NULL) { return NULL; } py_tensor_map_type = PyObject_GetAttrString(driver_mod, "PyCUtensorMap"); if (py_tensor_map_type == NULL) { return NULL; } PyObject *m = PyModule_Create(&ModuleDef); if(m == NULL) { return NULL; } PyModule_AddFunctions(m, ModuleMethods); return m; } ) values enumerater_BASE_ARGS_FORMATr}itemsrr{rmr|) constantsr~rrexpand_signature args_formatformatflat_signaturer args_list arg_decl_listrrl arg_declsinternal_args_listparamsnewline ptr_decls tma_declsfloat_storage_declsr<r)rrrrr make_launchers  '    ,     56rsv Srccs|]}||fVqdSr/r)rrrrrr\sr  c Cs|dur|jg|j|j|jdk|j|jS|d}|d}|d}|d}|d}|j}|j}|ddks;J|jdkrBdnd } |rRt|}|dd 9<tjjjj |j ||t ||||| } | g||S) Nnanswizzle elem_size elem_type block_size fp4_paddedr rsrrt) baserstridespaddinglisttritonruntimedriveractiveutilsrFdata_ptrTMA_DTYPE_DEVICE_TO_HOST) argmetadatarrrrrrrr cu_tensor_maprrrmake_tensordesc_argbs2*  rcsvtdd|D}|sStddt|Dr(ttks(Js1dgtfdd}|S)Ncss$|] }t|to |dVqdS)rPNrvr&rk)rrrrrrs"z)wrap_handle_tensordesc..cSs(g|]\}}t|tr|dr|qS)rPr)rrrrrrrs(z*wrap_handle_tensordesc..csft|dt}d}t|tdD]\}}|vr)|t|||d7}q||q|S)Nrrs)r_BASE_ARGS_FORMAT_LENrextendrr{)args final_argsrrrlaunchertensordesc_indicesrrrinners  z%wrap_handle_tensordesc..inner)r'rsetrr})rr~rhas_tensor_desc_argrrrrwrap_handle_tensordescs rc@seZdZddZddZdS) CudaLaunchercstdrjnt}fddfdd|D}ddjD}t|dd}t|||tdtt t d }t|d d |_ t |j |||_ |j|_|j|_|j|_|j|_|j|_|j|_dS) Nrcst|tr jj|fS|Sr/)rvr&fn arg_namesindex)r)r<rrsz'CudaLauncher.__init__..csi|] \}}||qSrrridxvalue)arg_idxrrrsz)CudaLauncher.__init__..cSrrrrrrrrrr__triton_launcherr;num_ctasrs)r4rdictrr~getattrrrr1r>r?rrlaunchglobal_scratch_sizeglobal_scratch_alignprofile_scratch_sizeprofile_scratch_alignlaunch_cooperative_grid launch_pdl)rGr<rrr~rrHr)rr<rrIs*    zCudaLauncher.__init__c sbfdd}|jjtj}|jjtj} j|jj || g |RdS)Ncs:|dkr}|j|}|}|||SdSNr)rget)sizealign allocator grid_size alloc_sizealloc_fngridXgridYgridZrGstreamrrallocate_scratchs   z/CudaLauncher.__call__..allocate_scratch) rrr _allocatorrr_profile_allocatorrrr) rGr r r r functionrr global_scratchprofile_scratchrrr__call__s  zCudaLauncher.__call__N)rJrKrLrIrrrrrrs rcsjeZdZfddZddZddZddZed d Zd e d e fd dZ ddZ ddZ ddZ ZS) CudaDrivercst|_t|_tdSr/)r2rr launcher_clsr5rI)rGr8rrrIszCudaDriver.__init__cCs6|}||}|dd|d}d}td||S)Nrrrs cuda)get_current_deviceget_device_capabilityr)rGdevice capability warp_sizerrrget_current_targets   zCudaDriver.get_current_targetcCsddl}|d|S)Nrr)torchrrrGrrrrget_active_torch_devicesz"CudaDriver.get_active_torch_devicecCsddl}|jSr)rrrrrrget_device_interfaceszCudaDriver.get_device_interfacecCs6zddl}|jo|jjduWStyYdSw)NrF)rr is_availableversionhip ImportError)rrrr is_actives  zCudaDriver.is_activerlreturncCst|Sr/)rm)rGrlrrrmap_python_to_cpp_typesz!CudaDriver.map_python_to_cpp_typecCsddlm}|S)Nr)do_bench)triton.testingr))rGr)rrrget_benchmarkers zCudaDriver.get_benchmarkercCs&ddl}d}|jt|d|jddS)Nrir)rr)remptyint)rGr cache_sizerrrget_empty_cache_for_benchmarksz(CudaDriver.get_empty_cache_for_benchmarkcCs |dSr/)zero_)rGcacherrr clear_caches zCudaDriver.clear_cache)rJrKrLrIrr r! staticmethodr&r&r(r+r0r3rMrrr8rrs   r), functoolsrr!rrwpathlibrrtriton.runtime.buildrtriton.runtimertriton.backends.compilerrtriton.backends.driverrrrrealpath__file__rr>r0r?rA lru_cacher.r1objectr2rmrrrr}rrrr|rrrrrrrrrs`         `'+