o پi1@s ddlZddlZddlmZmZddlZddlmZddlmZm Z m Z m Z m Z ddl mZej d#dejdedejdejd ejf d d Zej  d$dejdedejd ejdejd eed ejfddZej  d%dejdedejd eejd eedejd ejfddZej     d&dejejBdejdejedeejd eejd eedejdeed ejfddZej     d'dejdejedeejd eejdeed eeded e e eejgfddZej d(dejdeded ed ejf d!d"ZdS))N)CallableOptional)Int32Int64Float32Boolean const_exprvalopreduction_bufferinit_valreturnc Cs|tjtj}}t|jd}||||}}|dkr&||||f<tj|} ||kr7|||f} tj| |S)zDreduction_buffer has shape (num_warps / warp_per_row, warps_per_row)r)cutearchlane_idxwarp_idxsizeshapebarrierwarp_reduction) r r r r rr warps_per_rowrow_idxcol_idxblock_reduce_valr@/home/ubuntu/.local/lib/python3.10/site-packages/quack/reduce.py block_reduces   rmbar_ptrphasecCsJtj}tjtj}}|j\} \} } || || } } |dkrLtj| | }tj||| |jj dWdn1sGwY|| krat j |t || | |ff||dtjj ||durk|ndd|}t| | tjj}t|D]}||tjj}|tj|dgdkr|||| |f}qtj||S)zRreduction_buffer has shape (num_warps / warps_per_row, (warps_per_row, cluster_n))rNpeer_cta_rank_in_clusterr rmode)rrblock_idx_in_clusterrrr elect_onembarrier_arrive_and_expect_tx element_typewidthutilsstore_shared_remote elem_pointer mbarrier_waitceil_div WARP_SIZEcutlassrange_constexprrr)r r r rr r cta_rank_in_clusterrrrows_per_blockr cluster_nrr num_warpsrnum_iteriidxrrrcluster_reduces8  r;cCs0t|durt||||dSt||||||dS)zPPerform either block or cluster reduction based on whether mbar_ptr is provided.N)r r r )rrr;)r r r rr r rrrblock_or_cluster_reduceEs r=xthreads_per_rowhook_fnc Cstt|tjr|j||dd}n|}tjjtjtjj t|j t kr&tj j nttjjttjjtji|} tj j|| t|tj jd}t|durK|t|durx|jd\} } | dksd|dusdJdt| dkpl| dkrxt|| ||||d}|S)zXreduction_buffer must have shape (num_warps / warps_per_row, (warps_per_row, cluster_n))rr reduction_profilethreads_in_groupNr/mbar_ptr must be provided for cluster reductionr<)r isinstancer TensorSSAreduce ReductionOpADDoperatoraddMAXdtyperrfmaxmaxMINminMULmulrr1rr=) r>r r?r rr r r@r warp_oprr6rrr row_reduceUs6       rVF return_exp_xcCs|jtks Jd tjj|jtjjtj ddtjj t |tjj d}t t j}tj j||||dd} tjj| jtjjdddtjt |tjj d} t|durX|t|dur|j\} \} } | dksr|dusrJd t| dkpz| dkr|jtksJd tjtj}}|| || }}t|dur|dkrt|| |||f<tjtj }d} || krt|||f\}} tj|tjj }| tj j||dd9} tj| tj} t|r| tj j||dd9} |}ntj}|dkr+tj| | }tj ||| |jj!d Wdn 1s&wY|| krEtj"t|| t#||||ff||d tjj$||durP|ndd t%| | tjj }t&|t}|'tj t&|t}|'dt()|D]%}||tjj }|tj*|dgdkrt|||f\||<||<q{|+jtjjtj dd}tj|tjj }d} t()|D]}| ||tj j|||dd7} qtj| tj} t|r| tj j||dd9} |}|| t|r| fSdfS)Nzx must be of type Float32rrArCT)fastmathr rrEz+reduction_buffer must be of type cute.Int64r!r"r$r%),rNrrrrrHrIrMinfrOrRr1mathlog2eexp2rJrKrLrrr*rrrr, f32x2_to_i64r i64_to_f32x2expr'r(r)r+r-r.r/r0 make_fragmentfillr2r3rload)r>r?r rr@r rWmax_xlog2_eexp_x sum_exp_xr5rr6rrrrmax_x_single_warp max_x_finalr4r7r8sum_exp_x_single_warpr9r:rrronline_softmax_reduce~s                     rkr X elem_per_lane subwarp_size warp_sizec Cs|dkr|dkr|dtt|>ksJ|dkr*||dkr*|dtt|>ks,Jtj|}t|t|}tj|dgd}|d|ksLJt j tt|tt||ddD]%}t j t|ddD]}tjj ||d|>|d}|||||<qlq`t tttj|dgddddD]_} d| >} t j | ddD]Q} |d | f} |d | | f} t || @ }t j t| ddD]/}| || |}}|r|n|| |<|r|n|| |<tjj | || |d}| ||| |<qqq|d S) aX For warp reduction, we use Swap Shuffle The normal way to reduction among threads: use shuffle to let *** the first half of threads *** have *** whole data *** from the second half of threads. After each step of reduction, a half of threads won't work in the following steps. That is, as the reduction progresses, the efficiency of shuffle & reduction instructions gradually change from 1/2, 1/4 to 1/32 (the worst case). To overcome this shortcoming, for a NxN matrix to be reduced among N threads as a 1XN vectors, we use swap & shuffle aiming to let *** each half of threads *** have *** a half of data *** from the other half of threads. After reduction, each half of threads should deal with a (N/2)x(N/2) sub-matrix independently in the following step. We can recursively do this until the problem size is 1. rrlrr%T) unroll_full)offsetN)Nr)intrZr[rrrlogical_divide make_layoutrr2rangeshuffle_sync_bflyr3r)rmrnrorprnumvecr9vshfl_vallogmmrfrg_Afrg_B should_swaplowerupperrrrsum_swap_shuffles@  , r)r )r N)Nr )NNNr N)NNNNF)rrrl)rZrKtypingrrr2 cutlass.cuterrrrrr quack.utilsr,jitNumericTensorrPointerr;r=rGrI ConstexprrtrVboolrkrrrrrs    %   ( b