o i7@sZddlmZddlmZddlZddlmZddlmmZeddGdddZ dS))Optional) dataclassNT)frozenc@seZdZUejeed<ejeed<ejed<ejed<dZe ejed<dZ e ejed<dZ ejeed <e j dd e jd ejd ejde jdejedejedejeddfddZe j d e jd ejd ejde jde jdejdejdejddfddZdS) AttentionMask m_block_size n_block_sizeseqlen_qseqlen_kNwindow_size_leftwindow_size_rightqhead_per_kvhead_packgqaFacc_Sm_blockn_blockthr_mma mask_seqlen mask_causal mask_localreturncCs|r|rJdt|}t|j|jf} t|| } t|d| } | dd} |j||j| } t | o@| rt |rt j t | j dddD]/}| d|fd| k}t j t | j dddD]}|rvt jj n|||f|||f<qmqTdSdS|jj dd}t |jdkrtjj|dksJdt |j d|ksJ|j}||j| ||dfd|j}d|j||j|j| }t |rEt j t | j dddD]]}t |jdkr| |dfd||j}n tj||||d}||}t |rt || }t j t | j dddD]}| d|fd|kr6t jj n|||f|||f<q$qdSt |jdurS||jnd}t |jdure|d|jnd}t j t | j dddD]|}t |jdkr| |dfd||j}n tj||||d}t |jdur||}t |rt || }n|j}t |jdur||nd}t j t | j dddD]}| d|fd}||ks||krt jj |||f<qАqtdS)N.mask_causal and mask_local cannot be both Truerr T unroll_fullz%threads_per_row must divide WARP_SIZE)width)utilsmake_acc_tensor_mn_viewcutemake_identity_tensorrr partition_C get_slicer cutlass const_exprrangesizeshapeFloat32inf tv_layout_Cr arch WARP_SIZEthr_idxr shuffle_syncminr r )selfrrrrrrracc_S_mncStScS_mnt0ScS_mnthr_col_offsetseqlenk_col_limitcoobrthreads_per_rowtidx mma_m_idxcausal_row_offsetrow_idxcol_limit_rightlocal_row_offset_rightlocal_row_offset_leftcol_limit_leftcol_idxrAJ/home/ubuntu/vllm_env/lib/python3.10/site-packages/flash_attn/cute/mask.py apply_masks   $    8   zAttentionMask.apply_mask thr_tmem_loadc Cs|r|rJdt|j|jf} || } || } |j||j} t| o*| rt|rtt | j } t| ddk rctj | ddD]}| |d| krZtj j n||||<qKdStj | dddD]A}| |d}tt|d}td|>d}tdD] }t|d|>@}|r||d|ntj j ||d|<qqldSdSd|j||j|j}| dd||j}t|jdkr||j}t|rm||}t|rt|| }tt | j } t| ddk rtj | ddD]}| |d|krtj j n||||<qdStj | dddD]B}||d}tt|d}td|>d}tdD]"}t|d|>@}|r\||d|ntj j ||d|<qFq(dSt|jdur{||jnd}t|jdur|d|jnd}t|jdur||}t|rt|| }n|j}t|jdur||nd}tj t | j ddD]}| |d}||ks||krtj j n||||<qdS)NrrTrr )rrrrr partition_Dr r r!r#r$r"r%r&Uint32maxrange_constexprBooleanrr r,r r )r-rrrrrDrrrr/tScStScS_t2rr3ncoliscol_limit_right_scol_limit_right_curmask mask_i_bitr:r;r<r=r>r?r@rArArBapply_mask_sm100ws      ,    "  0     zAttentionMask.apply_mask_sm100)F)__name__ __module__ __qualname__r Constexprint__annotations__Int32r rr r rjitTensorTiledMmaboolrC TiledCopyrTrArArArBr s^     `  r) typingr dataclassesrr cutlass.cuterflash_attn.cute.utilsrrrArArArBs