o i @s*ddlZddlZddlmZddlmZddlmZmZm Z ddl Z ddl m Z m Z daedgd Zed gd dd Zd efddZdefddZeZddZeZddZejdefddZddZeGdddZdedefd d!ZdJd#e jd$ed%ed&efd'd(Z dJd#e jd$ed%ed&efd)d*Z!e j"j#d+d,d- "dJd#e jd$ed%ed&ede jf d.d/Z$e j"%d+dJd0d1Z&d2edefd3d4Z'd2edefd5d6Z(d2ed7eedefd8d9Z)de efd:d;Z* d?Z+dLdAdBZ,dCeeefdDdEZ-d@a.d efdFdGZ/defdHdIZ0dS)MN) dataclass)Enum)DictListTuple)ceil_divround_upF AuxStreamType) Attention MoeSharedMoeChunkingOverlap EventType)Mainr r r )startenablecC|adSNis_torch_compiling_flagrrP/home/ubuntu/vllm_env/lib/python3.10/site-packages/flashinfer/fused_moe/utils.pyset_torch_compilingrreturncCtSrrrrrris_torch_compilingrcCrr) _global_attrsrrrrget_global_attrs%sr cCs ttddSNattrs)getattr_model_extra_attrsrrrrget_model_extra_attrs,s r%r"ccs0ttdd}|t_z dVW|t_dS|t_wr!)r#r$r")r" old_attrsrrrmodel_extra_attrs0s  r'csfdd}|S)Ncsfdd}|S)NcsFt||g|Ri|WdS1swYdSr)r')selfargskwargs)func get_attrsrrwrapper<s$z:with_model_extra_attrs..decorator..wrapperr)r+r-r,)r+r decorator;sz)with_model_extra_attrs..decoratorr)r,r/rr.rwith_model_extra_attrs:s r0c@s:eZdZUejed<ejed<dZeed<eddZ dS)Fp4QuantizedTensor fp4_tensorscaling_factorTis_sf_swizzledcCs|jjSr)r2shape)r(rrrr5KrzFp4QuantizedTensor.shapeN) __name__ __module__ __qualname__torchTensor__annotations__r4boolpropertyr5rrrrr1Es   r1rowcolcCst|d}t|d}||fSN)r )r>r? padded_row padded_colrrrcompute_swizzled_sf_shapePs  rEsfrowscolsscaling_vector_sizecCs&t||}|d||}tjj|S)aSwizzle FP4 scaling factors using C++ torch op implementation Args: sf: [b, rows, cols_sf] or [rows, cols_sf]. The original unswizzled scaling factors. rows: rows of the original unquantized tensor cols_sf: ceil_div(cols, scaling_vector_size) where cols is the number of columns of the original unquantized tensor scaling_vector_size: the size of the scaling vector Returns: [b * round_up(rows, 128) * round_up(cols_sf, 4), ] 1D swizzled scaling factors, possibly with rows and cols padded. )rviewr9opstrtllmblock_scale_interleaverGrHrIrJsf_colsrrr swizzle_sfVs rRcCs.t||}|d||}tjj|d|S)a^Swizzle FP4 scaling factors using C++ torch op implementation Args: sf: The (padded and) swizzled scaling factors. rows: rows of the original unquantized tensor cols: cols of the original unquantized tensor scaling_vector_size: the size of the scaling vector Returns: 2D unswizzled scaling factors rK)rrLr9rMrNblock_scale_interleave_reverserPrrr unswizzle_sfes rTztrtllm::reswizzle_sfr) mutates_argsc Cst||}t||\}}||}|||dksJ|||}||||} t| |||} ||} | |||} | ddd|d|f} | | |} t| | ||S)aReswizzle FP4 scaling factors using C++ torch op implementation. It unswizzles the scaling factors in each partition first, then concatenates them together, and finally swizzles them back. Args: sf: The (padded and) swizzled scaling factors. rows: rows of the original unquantized tensor cols: cols of the original unquantized tensor scaling_vector_size: the size of the scaling vector Returns: 1D reswizzled scaling factors rN)rrEnumelrLrT contiguousrR) rGrHrIrJrQ padded_rowspadded_sf_cols padded_colsnum_partitions sf_reshaped sf_unswizzled total_rowssf_concatenatedrrr reswizzle_sfts$  r`c CsNt||}t||\}}|||}||}t|dt|d} || Sr@)rrErVr new_empty) rGrHrIrJrQrXrYr[r^szrrr_s  rcxcCsd|dkrdS|d}||d?O}||d?O}||d?O}||d?O}||d?O}||d?O}|dS)NrrBrF r)rdnrrrnext_positive_power_of_2s      ricCst|}||kr |S|dSNr)ri)rdnextrrrlast_positive_power_of_2srlbucketscCsttt||d|dS)NrrK)minmaxri)rdrmrrrnearest_in_bucketssrpcCs:t|}g}|}|dkr|||d}|dks t|S)Nrer)riappendtuple)max_num_tokensnum_token_bucketsmrrr!get_power_of_2_num_tokens_bucketss rvre.cCs:t|}g}|}||kr|||d}||ks t|Srj)rlrqrr)rsmin_num_tokensrtrurrr&get_last_power_of_2_num_tokens_bucketss rxTcCs|d}tt|dD]}|||9}q dd|D}|dd<|r2t|dt|d|dn||d|}||fS)NrecSsg|]}|qSrr).0irrr sz!get_fp4_shape..rKrrArB)rangelenr ) input_shape sf_vec_sizeis_swizzled_layoutrurz output_shape scale_shaperrr get_fp4_shapesr input_shapescCst|ddd\}}|dS)z1Calculate the dimensions of the fp4 scale tensor.rrF)rr)r)r out_shaperrrrfp4_scale_infer_shapesrcCrr_enable_piecewise_cuda_graphrrrrset_piecewise_cuda_graph_flagrrcCrrrrrrrget_piecewise_cuda_graph_flagrr)rF)re)T)1 contextlib threading dataclassesrenumrtypingrrrr9utilsrr rr rr<rrlocalrr r$r%contextmanagerr'r0r1intrEr:rRrTlibrary custom_opr` register_fakercrirlrprvrxrrrrrrrrrsv      '