o پi"@s ddlmZmZmZddlmZddlZddlmm Z ddlm Z ddl m Z mZddlmZmZddlmZddlmZdd lmZdd lmZdd lmZd d e j dd ee j!dddZ"dd dd dd dd dd dZ#ee$dZ%ddZ&de'fddZ(eddee%dDd gd!e(id"   # #     $ $ d{d%e d&e d'e d(ee d)ee d*e)e Bd+e)e Bd,ee d-ee d.ee d/ee d0e*d e*d1ee d2dfd3d4Z+ed5dee%dDd6d gd!e(id"      $ d|d%e d&e d7ee d8e d(ee d)ee d6edd,ee d.ee d e*d1ee d2dfd9d:Z,ed;dee%dDd6d gd!e(id"    < d}d%e d&e d=e d>e d8e d6edd,ee d.ee d e*d1ee d2dfd?d@Z-   #      $ ee d8ee dAeej.dZeej.d,ee d.ee d e*dBe*d2ee e ffdfdgZej/j0dhdiddjda    < e d8e d6edd,ee d.ee d e*dBe*d2dfdkdlZ:     dd%e d&e d=e d6edd,ee d.ee dAeej.dZeej.d2ee e ffdmdnZ;   o        $ # #dd%e d&e d(ee d'ee dAeej.d e*d*e)e Bd+e)e Bd2eee e ffdydzZdS))OptionalTupleLiteral)partialN)Tensor) GemmConfigget_all_configs)autotuneAutotuneConfig)get_device_capacity)gemm)gemm_act) gemm_dact)gemm_symmetriccCs|SNxrrH/home/ubuntu/.local/lib/python3.10/site-packages/quack/gemm_interface.pysrcCst|Sr)Frelusquarerrrrrtanh approximate)Nrrelu_sqgelu_tanh_approxcCt||Sr)rsilugateuprrrrrcCs|td||dS)NgZd;?torchsigmoidr!rrrr scCrr)rrr!rrrr!rcCstj|dd|S)Nrr)rgelur!rrrr"scCrrr%r!rrrr#r)swiglu swiglu_oaireglugegluglucudacCs4t|ddkrtddddddStd d ddd dS) Nr r$T)tile_mtile_n cluster_m cluster_npingpongF)r r)devicerrrdefault_config*sr: named_argscKsj||B}|dddu}|dddu}|s|rdd|D}|r3t|djddkr3dd|D}|S) NA_idx cu_seqlens_mcSsg|] }|jdjs|qSconfig)kwargsswap_ab.0confrrr 6sz.prune_invalid_gemm_configs..Ar cSs0g|]}|jdjdkr|jdjdkr|qS)r?r$)r@r6r4rBrrrrE:s  )getr r9)configsr;r@gather_Avarlen_mrrrprune_invalid_gemm_configs1srMcCg|]}t|dqSr>r rCcrrrrECrEdynamic_schedulerearly_config_prune)rJkeyprune_configs_by?FrFBoutCbiasalphabetar= cu_seqlens_kr<batch_idx_permute add_to_outputr?returncCs"| dur t|j} |du}|du}|p|}| du}|r*|s!Jd| jdks*Jd|r3| jr3Jd|jdkr?|s?|d}|j}|jdkrN|sN|d}|dur^|jdkr^|s^|d}|jdkrj|sj|d}|durx|jdkrx|d}|s|jdn|jdd}|r| dur| jdn|jd}||jdf}n ||jd|jdf}|j|ksJd|jd || rtj dtj |jd nd}t | js|n|| js|n|| js|n|j|dur| js|n|jnd|| j | j | j| j| jf d | j| js|nd| jr|nd||||| | | d dS) Nz-gather_A requires either varlen_m or varlen_kr$zgather_A requires cluster_n=14Variable-length sequences not supported with swap_abr2rzout shape mismatch: z vs dtyper9T) persistentmax_swizzle_size rowvec_bias colvec_biasr]r^r=r_r<r`ra)r:r9r6rAndim unsqueezemTshaper&zerosint32gemm_sm90_sm100r3r4r5r7rh)rFrYrZr[r\r]r^r=r_r<r`rarTr?rLvarlen_kvarlenrK batch_sizetotal_m out_shapetile_count_semaphorerrr gemm_tunedBsl             rxcCrNrOrPrQrrrrErS activation preact_out postact_outc Cs| dur t|j} |du} | r| jrJd|jdkr"| s"|d}|j}|jdkr/|d}|dur?|jdkr?| s?|d}|durP|jdkrP| sP|d} n|} |jdkr_| s_|d} n|} |duro|jdkro|d}| r{tjdtj|jdnd}t | js|n|| js|n|| dur| js| n| jnd|dur| js|n|jnd| js| n| j||| j | j | j | j | jd| j| js|nd| jr|nd||ddS)Nrcr2rr$reT)rgrhrirjr=r<)r:r9rArkrlrmr&rorpgemm_act_sm90_sm100r3r4r5r6r7rh)rFrYrzr{r[r\ryr=r<rTr?rLDPostActrwrrrgemm_act_tunedsT             rcCrNrOrPrQrrrrErSTPreActdx_outc Cs>| dur t|j} |du} | r| jrJd|jdkr"| s"|d}|j}|jdkr/|d}|jdkr;| s;|d}|jdkrH| sH|d} n|} |jdkrW| sW|d} n|} |retjdtj|jdnd} t | jsm|n|| jss|n|| jsy| n| j| js|n|j| js| n| j| || j | j | j | j | jd| j||ddS)Nrcr2rr$reT)rgrhr=r<)r:r9rArkrlrmr&rorpgemm_dact_sm90_sm100r3r4r5r6r7rh)rFrYrrr{ryr=r<rTr?rLr}r~rwrrrgemm_dact_tunedsL          r out_dtypetunedc Cs|durh|dur |jn|}|du} |du} | r-|dur |jdn|jd}||jdf}n2| rB|jdd}||jd|jdf}n|jdkrQ|jd|jdfn |jd|jd|jdf}tj|||jd}t|tso|nd}t|trx|nd}t|||||||||| | | d |S) z4GEMM with optional output tensor and tuning control.Nrr$r2rdrerX) r\r] alpha_tensorr=r_r<r`rTr) rfrnrkr&emptyr9 isinstancefloatgemm_out)rFrYrZr\r]rr=r_r<r`rTrrLrrrurvLrrrrr s<8r zquack::gemm_outrZ) mutates_args device_typesrc CsH| rtnttjdd} |dur|n|}| |||d|||||| | d dS)z&GEMM with pre-allocated output tensor.Nr>)r[r\r]r=r_r<r`rTrxrfn) rFrYrZr\r]rr=r_r<r`rTrrrrrrEs rc  Cs|dur|jn|}|durG|durG|jdkrtjntj} | ||||d}t|tr-|dkr1||9}|durE|jdkr<|n|d}||7}|S|dur|durk|durX|jdn|jd} tj | |jdf||j d }t |jddD]c} |dur|||| || dn ||| || d} tj| || ||| || dd t|tr|dkr||| || d|9<|dur||| || d|| 7<qt|S|jdd} |durtj | |jd|jdf||j d }t | D]A} |dur|dd||| || dfn|dd|| || df} tj| ||| || dddf|| d qt|trH|dkrL||9}|durU||7}|S) zr]r^r=r_r<r`rarTr)rFrYr[rZr]r^rrr=r_r<r`rarTrrrrrrs" rc  Cs|dur[|dur[t|trt|trtj|||| |||d}n&|dur&|jn| dur,| n|j} |||||| } |durE|| |durY|jdkrP|n|d}||7}|S|dur|dur| durl| j dn|j d} | durw| n|j} tj | |j df| |j d}t |j ddD]U} | dur|| || || dn ||| || d}||| || d}||| || d}|t ||| ||} |dur| || 7} || q|S|j dd}| dur| n|j} |durtj ||j d|j df| |j d}t |D]O} | dur-|dd| || || dfn|dd|| || df}||| || dddf}|t ||||| } || | q|durm||7}|S)zIReference implementation for GEMM with addition and pre-allocated output.N)rr]r^rZr2r$rrre)rrr&addmmrftocopy_rkrlrnrr9rr)rFrYr[r\rZr]r^r=r_r<rresultrurrC_slice out_slicerB_slicerrr gemm_add_refsX %   $  $  rc Csnt|ts|nd} t|tr|nd}t|ts|nd} t|tr"|nd}t|||||| | ||||| | d dS)aIn-place GEMM with addition: out = alpha * A @ B + beta * out. Args: A: (M, K) or (L, M, K) or (total_M, K) if varlen_m or (M, total_K) if varlen_k - input tensor B: (K, N) or (L, K, N) or (total_K, N) if varlen_k - input tensor out: (M, N) or (L, M, N) or (total_M, N) if varlen_m or (L, M, N) if varlen_k - tensor to accumulate into (modified in-place) alpha: Scalar multiplier for A @ B beta: Scalar multiplier for out cu_seqlens_m: Optional cumulative sequence lengths for variable M cu_seqlens_k: Optional cumulative sequence lengths for variable K dynamic_scheduler: Whether to use dynamic scheduler tuned: Whether to use autotuned configuration NrX)r<r`rTr)rrgemm_add_inplace_op) rFrYrZr]r^r=r_r<r`rTrrrrrrgemm_add_inplaceTs& rzquack::gemm_add_inplacec Cs|| rtnttjdd} |dur|n|}|dur|n|}t|to'|dko'|du}| ||||s0|nd||||| | || d dS)Nr>rXr)rxrrrr)rFrYrZr]r^rrr=r_r<r`rTrrrarrrrs$  r postact_dtype store_preactc Cs|dur|jn|}|dur|jn|}| du}|r.| dur!| jdn|jd}||jdf}n|jdkr>|jd|jdf}n|jd|jd|jdf}|dur[| r[tj|||jd}|durhtj|||jd}t|||||||| | | | ||fS)z1GEMM with activation and optional output tensors.Nrrr2rdre)rfrnrkr&rr9 gemm_act_out)rFrYr[r\ryrzr{rrr=r<rrTrrLrurvrrrr s6   r zquack::gemm_act_out)rzr{z(Tensor A, Tensor B, Tensor(a2!)? preact_out, Tensor(a3!) postact_out, Tensor? C=None, Tensor? bias=None, str? activation=None, Tensor? cu_seqlens_m=None, Tensor? A_idx=None, bool dynamic_scheduler=False, bool tuned=True) -> ())rrschemac Cs4| rtnttjdd} | |||||||||| dS)z6GEMM with activation and pre-allocated output tensors.Nr>)rrr) rFrYrzr{r[r\ryr=r<rTrrrrrrsrc Cs|dur|jn|}|dur|jn|}|dur t|||||d} n t||||||d} t|| |} | r<| || fSd| fS)Nr\r=r<)rfrract_to_pytorch_fn_mapr) rFrYr[r\ryr=r<rrrrZpostactrrr gemm_act_refs rc  Cs|dur|jn|}|dur|jn|}|du} | r.| dur!| jdn|jd} | |jdf}n|jdkr>|jd|jdf}n|jd|jd|jdf}|durYtj|||jd}|durftj|||jd}t|||||||| | | ||fS)z:GEMM with activation gradient and optional output tensors.Nrrr2rdre)rfrnrkr&rr9 gemm_dact_out)rFrYrryrr{rrr=r<rTrrLrurvrrrrs" rzquack::gemm_dact_out)rr{z(Tensor A, Tensor B, Tensor PreAct, Tensor(a3!) dx_out, Tensor(a4!) postact_out, str? activation=None, Tensor? cu_seqlens_m=None, Tensor? A_idx=None, bool dynamic_scheduler=True, bool tuned=True) -> ()c Cs2| rtnttjdd} | ||||||||| dS)z?GEMM with activation gradient and pre-allocated output tensors.Nr>)rrr) rFrYrrr{ryr=r<rTrrrrrr2src Cs|dur|jn|}|dur|jn|}t||||d|}t||} |dur*|} n|j} |dt||} tjj| ||ddd} || | || |fS)z;Reference implementation for GEMM with activation gradient.Nr=r<TF create_graphr) rfrrr requires_gradrequires_grad_r&autogradgrad) rFrYrryr=r<rrdoutrdxPreAct_requires_gradpostact_for_gradrrr gemm_dact_refIs     rr))r-r)r*r+r,c Cs|dur|jn|}|dur|jn|}|dur t|||||d} n t||||||d} | ddddf} | ddddf} t|| | } | rJ| |nd| |fS)aKReference implementation for GEMM with gated activation forward. Args: A: (M, K) - input tensor B: (K, N) - weight tensor with gate and up projections C: (M, N) - optional bias tensor activation: Type of gated activation out_dtype: Output dtype for preact postact_dtype: Output dtype for postact store_preact: Whether to return the pre-activation Returns: (preact, postact) where: - preact: (M, N) pre-activation (if store_preact=True, else None) - postact: (M, N // 2) post-activation output Nr.r2r$)rfrrgated_to_pytorch_fn_mapr)rFrYr[r\ryr=r<rrrpreactr"r#rrrrgemm_gated_refdsrcCs|dur|jn|}|dur|jn|}t||||d|}|ddddf} |ddddf} | j| j} } | d| dt|| | } tjj| | | g|dd\}}| | | | tj ||gd d  |j }||| |fS) aReference implementation for GEMM with gated activation gradient. Args: A: (M, K) - dout input tensor B: (K, N) - weight tensor PreAct: (M, 2*N) - pre-activation tensor with gate and up projections interleaved activation: Type of gated activation out_dtype: Output dtype for dx postact_dtype: Output dtype for postact Returns: (dx, postact) where: - dx: (M, 2*N) gradient w.r.t. PreAct - postact: (M, N) post-activation output Nr.r2r$TFrr)dim) rfrrrrrr&rrstackreshapern)rFrYrryr=r<rrrr"r#gate_requires_gradup_requires_gradrdgateduprrrrgemm_dgated_refs    rzquack::gemm_symmetric_outzz(Tensor A, Tensor B, Tensor(a2!) out, Tensor? C=None, bool dynamic_scheduler=False, float alpha=1.0, float beta=1.0) -> ()cCs|jdkr |d}|j}|jdkr|d}|dur%|jdkr%|d}|jdkr0|d}n|}|r>tjdtj|jdnd}t|||durI|nd|durP|nd|dddddd d ||d dS) z&GEMM with guaranteed symmetric output.r2rNr$rer0r8FT) tile_Mtile_N cluster_M cluster_Nr7rgrhr]r^)rkrlrmr&rorpr9gemm_symmetric_sm90_sm100)rFrYrZr[rTr]r^rwrrrgemm_symmetric_outs8        rc Cs|dur|jn|}|jdkr|jd|jdf}n|jd|jd|jdf}|dur4tj|||jd}t|tr;|nd} t|trD|nd} t|||||| | d|S) zGEMM with symmetric output.Nr2rrrdrerX)rTr]r^) rfrkrnr&rr9rrr) rFrYr[rZrrTr]r^rv alpha_valbeta_valrrrrs r) NNrXrXNNNNFFN)NNNNNFN)NNNTN) NNrXNNNNNFT) NrXNNNNNFT)NNrXNNNN) NrXrXNNNNNFT) rXrXNNNNNNFFT)NNrXrXNNNN)rXrXNNNNFT) rXrXNNNNNNFT) NNNNNNNNNTFT)NNNNNFT)NNNNNNNT) NNNNNNNTT)NNNTT)NNNNN)NNr)NNNNT)NNNN)NFrXrX)NNNFrXrX)?typingrrr functoolsrr&torch.nn.functionalnn functionalrrquack.gemm_configrrquack.autotunerr r quack.cute_dsl_utilsr quack.gemmr rqquack.gemm_actr r|quack.gemm_dactrrquack.gemm_symmetricrrrr(rrr9default_device_capacityr:dictrMrboolrxrrrflibrary custom_oprrrrrrrrrrrrrrrrrrs                 K       >     <      3       %   ?      ;      +     H     /       (      0             $        . ,  -