o پi@sddlZddlZddlZddlZzddlmZeD]\ZZeej vr)eej e<qWn e y4Ynwddl m Z de fddZdad:d d Zd e fd d ZedZedZedZedZedZedZedZedZedZedZedZedZedZeZedZ edZ!edZ"edZ#edZ$ed Z%ed!Z&ed"Z'ed#Z(ed$Z)ed%Z*ed&Z+ed'Z,ed(Z-ed)Z.ed*Z/ed+Z0ed,Z1ed-Z2ed.Z3ed/Z4ed0Z5ed1Z6ed2Z7ed3Z8dd4l m9Z9dd5l m:Z:dd6l:Td7d8Z;e e;dS);N)persistent_envs) deep_gemm_cppreturncCstjdp tjd}|durTz0ttjd }tjddg|dd}tj tj |}Wdn1s:wYWnt ySd}tj |sQd}Ynw|dusZJ|S) N CUDA_HOME CUDA_PATHwwhichnvcc)stderrz z/usr/local/cuda) osenvirongetopendevnull subprocess check_outputdecoderstrippathdirname Exceptionexists) cuda_homerr rF/home/ubuntu/.local/lib/python3.10/site-packages/deep_gemm/__init__.py_find_cuda_homes    rFcCs6trdStjtjt}tjj |t dadS)NT) _dg_initializedr rrabspath__file__torchops deep_gemminitr) library_rootrrr_ensure_initialized%s r%namecsttjj|fdd}|S)Ncst|i|S)N)r%)argskwargsfuncrr_fn0sz_wrap_op.._fn)getattrr r!r")r&r+rr)r_wrap_op.s r- set_num_sms get_num_smsset_compile_modeget_compile_mode set_tc_util get_tc_util fp8_gemm_nt fp8_gemm_nn fp8_gemm_tn fp8_gemm_tt m_grouped_fp8_gemm_nt_contiguous m_grouped_fp8_gemm_nn_contiguousm_grouped_fp8_gemm_nt_masked k_grouped_fp8_gemm_nt_contiguous k_grouped_fp8_gemm_tn_contiguous bf16_gemm_nt bf16_gemm_nn bf16_gemm_tn bf16_gemm_tt!m_grouped_bf16_gemm_nt_contiguous!m_grouped_bf16_gemm_nn_contiguousm_grouped_bf16_gemm_nt_masked!k_grouped_bf16_gemm_tn_contiguouscublaslt_gemm_ntcublaslt_gemm_nncublaslt_gemm_tncublaslt_gemm_ttfp8_gemm_nt_skip_head_midfp8_mqa_logitsget_paged_mqa_logits_metadatafp8_paged_mqa_logitseinsum!transform_sf_into_required_layoutget_tma_aligned_size&get_mk_alignment_for_contiguous_layoutget_mn_major_tma_aligned_tensor,get_mn_major_tma_aligned_packed_ue8m0_tensor6get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor)testing)utils)*csFgd}ttjjjfdd|D}|r!td|dSdS)N)r#r.r/r2r3r4r5r6r7r8r9r:r;r<rNrOrPrQrRrSrIrJrKrLrMrErFrGrHcsg|]}|vr|qSrr).0op available_opsrr sz&_verify_ops_loaded..zWarning: Missing operations: )listr r!r"__dict__keysprint) expected_ops missing_opsrrYr_verify_ops_loadedps rbT)rN)r?r@rArBrCrDrErFrGrHrIrJrKrLrMrNrOrPrQrRrSrTrUrbrrrrs