o پiE@sdZddlZddlZzddlZeedredkredddlTddlTddl TWney>ddl Tddl Tddl TYnwz ddl mZmZmZWney^ddl mZmZmZYnweeZd d eDZd d ZdddZdddZddZddZddZddZdS)zN Utilities for selecting CUTLASS library kernels based on problem description NCUTLASS_IGNORE_PACKAGETz+Disabling attempt to import cutlass_library)*)get_valid_schedules)generate_data_types_from_math_instructionfix_alignmentscCsi|]\}}||qSr.0kvrrm/home/ubuntu/.local/lib/python3.10/site-packages/flashinfer/data/cutlass/python/cutlass_library/heuristics.py Bsrc Cs|}|D]K}|D]\}}t|trt|||<q t|tr't|||<q |d}|D]"}|D]\}}t|trDt|||<q4t|trOt|||<q4q.qt|d}tj ||ddWddS1skwYdS)a  Utilitiy function to write heuristics results to a json file for debug args: problems_with_configs: List of problems provided to the heuristic, with a list of operations added to each problem dict outfile_path: Outfile path returns: None configsw)indentN) copyitems isinstanceDataType DataTypeNames LayoutTypeShortLayoutTypeNamesopenjsondump) problems_with_configs outfile_pathpc_copypr r rcfrrr $serialize_heuristics_results_to_jsonDs*        "r#Fc Cs.| durt} | j|||||||||| | d S)a Get heuristic-suggested GEMM kernel configurations for a single GEMM problem. args: m, n, k: GEMM dimensions batch_count: batch count layouts: tuple of layouts of type LayoutType use_fast_acc: Use fast accumulation for FP8. Ignored for other precisions count: Number of configs to return provider: Heuristics provider to use returns: A list of dictionaries containing the suggested kernel configurations and additional info from the input required to define a Cutlass GemmOperation, with the following keys: - 'cta_tile_m', 'cta_tile_m', 'cta_tile_k': CTA tile size - 'instr_tile_m', 'instr_tile_n', 'instr_tile_k': Instruction tile size - 'stages': kernel pipeline stage count - 'cluster_m', 'cluster_n', 'cluster_k': cluster size - 'layout_a', 'layout_b': input tensor layouts of type LayoutType - 'alignment_a', 'alignment_b': input tensor alignments, in count of elements - 'dtype_a', 'dtype_b', 'dtype_acc': dtypes of a, b, and accumulator, of type DataType - 'swizzle_size' : suggested threadblock swizzle - 'split_k_slices': number of partitions of the k dimension for splitK - 'raster_order': raster order for CTAs over output tiles ('along_m' or 'along_n') N)voidC use_fast_acccount)MatmulHeuristics get_configs) mnr batch_countlayoutsdtypes alignment_a alignment_br$r%r&providerrrr get_single_gemm_config`s r1cCsg}|D]}|}z|d}|d}|d}|d}|d} |d} |d} WntyA} z td| d |d } ~ ww|d d } |d d}|dd}|dd }|dd}|dd}|dd}| ttjkrztd| t | dkrt dd| Dstd| t dd| D}z$| | | |d ur| n| | g}t dd|D}Wnty}z td|d }~ww|d d!t |d"}|d#d!t |d}t|||||||||dk||| }||d$<||q|S)%a Get heuristic-suggested GEMM kernel configurations for a set of GEMM problems. args: problems: List of dictionaries describing GEMM problems with the following keys: - 'm', 'n', 'k': Matrix dimensions (required) - 'dtype_a': Data type of matrix A (required) - 'dtype_b': Data type of matrix B (required) - 'dtype_c': Data type of matrix C (default: None) - 'dtype_d': Data type of matrix D (required) - 'dtype_acc': Compute data type (default 'f32') - 'layout': Operation layout (e.g. 'tnt') - 'alignment_a': Memory access granularity of A, in units of elements (default: 16 bytes equivalent elements) - 'alignment_b': Memory access granularity of B, in units of elements (default: 16 bytes equivalent elements) - 'alpha': Scalar multiplier for A*B (default: 1.0) - 'beta': Scalar multiplier for C (default: 0.0) - 'batch_count': Number of GEMM operations in batch (default: 1) - 'use_fast_acc': Enable fast accumulation for FP8 on Hopper (default: True) provider: Heuristics provider to use count: Number of configurations to return per problem (defualt: 1) returns: A copy of the input dictionary, with key `configs` added containing the selected gemm configs r)r*r dtype_adtype_bdtype_dlayoutzMissing required parameter z for problem N operationgemmr+r dtype_accf32dtype_calphag?betagr%TzUnsupported operation css|]}|dvVqdS)ntNr)r r!rrr z#get_gemm_configs..zDlayout must be a 3-character string containing only 'n' or 't', got css$|] }|dkr tjntjVqdS)tN)rRowMajor ColumnMajor)r lrrr r?s"css|]}t|VqdS)N) dtype_map)r dtrrr r?r@zUnsupported data type: r.rr/r)rKeyError_LOGGERerrorgetOperationKindNames OperationKindGemm ValueErrorlenalltuplelower DataTypeSizer1append)problemsr0r&retproblemr)r*r r2r3r4r5er6r+r8r:r;r<r%r, dtype_listr-rFr.r/rrrr get_gemm_configs}sT         2" r[c Csd}d}|dur t}g}g}|D]}|d|dg|d|dg|dd t|d gf}|d |d |d |d|d f\} } } } } |dddk}|rTd|dn|d|d|ddg}t|| | | tjtj}|j|j|drwt j n|j | |j |j dg}|d|rdnd|d|df}t |d|d|d|d|dd|dgdgd||||d|d|dfd}g}|r| tjtjgn | tjtjgt||g|g||tjtjgtjdD] }| || |qq||fS)+ Generate CUTLASS operations based on the list of configs provided by the heuristic provider args: manifest: manifest argument to which to add operations, or None to just return the operations without a manifest (for pruning an existing manifest) cuda_version: Cuda compiler version for generating cutlass operations kernel_configs: list of configs generated by the heuristic returns: (configs, operations): a list of heuristic-provided kernel configs along with a one-to-one corresponding list of the generated operations deNlayout_ar.layout_br/layout_drGr4r2r3r8r: cluster_mrr cta_tile_m cta_tile_n cta_tile_kr$)a_typeb_typec_typed_typeacc_typeepi_typer cluster_n cluster_k)rfrr cluster_shape)tile_schedulers gemm_kind)ManifestrTMathInstruction OpcodeClassTensorOp MathOperation multiply_add element_a element_brvoidelement_accumulatorTileDescriptionrUKernelScheduleTypeTmaWarpSpecialized2SmSm100EpilogueScheduleTypeTmaWarpSpecialized2SmTmaWarpSpecialized1SmSm100TmaWarpSpecialized1SmCreateGemmUniversal3xOperatorTileSchedulerTypeDefaultStreamKGemmKind Universal3x)manifest cuda_versionkernel_configsmin_ccmax_ccr operationsconfigr5ryrzr| element_c element_dis_2sminstruction_shapemath_instruction data_typestile_multipliertile_description schedulesorrr &generate_sm100_from_heuristics_configss^ 6,* "  (  rc Csd\}}|dur t}g}g}|D]}|dt|ddko*|dt|ddk}|d|dg|d |dgtjd gf} |d|d|d |d |d f\} } } } }gd}t|| | | tjtj}t || |d}|rtt || dd} gd}t |d|d|dgd|||||d|d|dfd}t ||||d| t j|dd\}}t|rt|| g|g||t jdD] }||||qt|rt|| g|g||tjgdD] }||||qq||fS)r\)ZrNr.r2rGr/r3r_r`rr8r:r4)rrr)element_source element_dest)alignment_bitsrcrdrerrbrmrnroi(#r%)rr is_alignedrinstantiation_levelr5rrenable_fp8_fast_acc)rr)rq)rsrTrrCrtrurvrwrxrrr}rrrrPrrUrr)rrrrrrrrrr5ryrzr|rrdummy_instr_shaperrdummy_warp_countrrstream_k_schedulesrrrr %generate_sm90_from_heuristics_configssh 0(,       rcsg}t|jd }t|}Wdn1swY|jdks'|jdkr)dn|j}t|d}tdd|jdDrC| d t |||j d }g}g}|D]Y} td d|jdDrpt |j rfdn||j| d \} } td d|jdDrt|j rdn||j| d \} } || 7}dd| Dfddt| | D} || 7}qQ|D] } |d| dq|stdt||j|S)a Prune a manifest according to heuristics suggestions from the problems file args: manifest: Cutlass manifest to prune args: generator.py args, requires: - args.heuristics_problems_file - args.heuristics_gpu - args.heuristics_testlist_file returns: A list of dictionaries, each of which has information about an operation and a problem from the input problems rNauto)gpucs|]}d|vVqdS)100Nrr archrrr r?or@z.;@)r0r&csr)90Nrrrrr r?vr@rcss |] }d|vp d|vVqdS)r101Nrrrrr r?xscSsi|] \}}|dkr||qS)rrr rrr r|sz=filter_manifest_and_write_heuristics_file..cs$g|]\}}d|i|qS)operation_name)procedural_name)r r!rproblem_without_configsrr }s$z=filter_manifest_and_write_heuristics_file..^$z!No valid configurations generated)rheuristics_problems_filerloadheuristics_gpur'any architecturessplit set_cta_div_nr[heuristics_configs_per_problemrheuristics_restrict_kernelsrrrzipadd_kernel_filterr Exceptionwrite_profiler_testlist_to_csvheuristics_testlist_file)rargsheuristics_problemsr"rmmhrall_configs_and_operationsrrXproblem_configsproblem_operationswith_problem_sizer6rrr )filter_manifest_and_write_heuristics_file\s4       rc Cs|}|D]"}|D]\}}t|trt|||<q t|tr't|||<q qt|ddd}|d}t j ||d}| | |WddS1sRwYdS)a( Write a list of configs to a testlist to be consumed by cutlass_profiler args: configs_list: List of kernel configs along with runtime arguments and any other columns to include in the CSV, expressed as a list of dictionaries outfile_path: Outfile path returns: None rr)modenewliner) fieldnamesN) rrrrrrrrkeyscsv DictWriter writeheader writerows) configs_listrprofiler_testlistr!r r ofilek_nameswriterrrr rs      "r)FTrN)Nr)__doc__rrbuiltinshasattrr ImportErrorcutlass_library.librarycutlass_library.generator#cutlass_library.heuristics_providerlibrary generatorheuristics_provider sm90_utilsrrrlogging getLogger__name__rIrrrEr#r1r[rrrrrrrr s:        JHM +