o Iii@sddlmZmZddlZddlmZddlZddZ      d(d d Z   d)d d Z Gdddej j Z Gdddej j Z Gdddej j Z     d*ddZ       d+ddZ       d+ddZd,ddZ                    d-deeeejfdeejdeejdeejd eejd!eejd"eed#eejd$eejd%eejfd&d'ZdS).)OptionalUnionNcCs"|dur|ddkr|S|S)N)stride contiguous)xr O/home/ubuntu/.local/lib/python3.10/site-packages/hopper/flash_attn_interface.pymaybe_contiguouss"r rrTrc CsZ|dksJddd||||fD\}}}}|ddkr)|ddkr)|n|}dd|||fD\}}}d d| | fD\} } d d| ||fD\} }}d d||fD\}}tjg|||||||||| | | | | ||||||||||d|d||||||R^}}}||g|RS) Nr#sink_token_length not supported yetcSg|]}t|qSr r .0rr r r 3z'_flash_attn_forward..rrcSrr rrr r r r5cSrr rrr r r r8rcSrr rrr r r r9rcSrr rrr r r r<r)rrflash_attn_3_cudafwd) qkvk_newv_newout cu_seqlens_q cu_seqlens_kcu_seqlens_k_new seqused_q seqused_k max_seqlen_q max_seqlen_k page_table kv_batch_idx leftpad_k rotary_cos rotary_sin q_descale k_descale v_descale softmax_scalecausal window_sizesink_token_lengthsoftcaprotary_interleaved num_splitspack_gqa sm_margin softmax_lserestr r r _flash_attn_forwards(       !r9FcCs|dksJddd|||||fD\}}}}}t||||||| | ||||| | | |||d|d||||^} } }}}| | ||fS)NrrcSrr rrr r r r{rz(_flash_attn_backward..r)rbwd)doutrrrrr7rr sequed_qsequed_kr$r%dqdkdvr.r/r0r1r2 deterministicr6 softmax_dr8r r r _flash_attn_backwardas8" rCc@s2eZdZe      d ddZedd ZdS) FlashAttnQKVPackedFuncNr rr Fc  Cs|dur |jdd}|dkr$|jddksJ|jdd\} } }n1|dks,J| dus2J|jd| d}|d| |jdksHJ|j| ||gd d\} } }t| | ||||||||| d \}} } }}}|| | |||||_||_||_||_ | |_ | |_ ||_ |S) Nrrdim)r/r+r,r-r0r1r2) shaperIunbindsplitr9save_for_backwardr.r/r0r1r2rAndim)ctxqkvr.r/r+r,r-r0r1r2rA num_heads_qrrr num_heads_kr out_paddedr7r r r forwards:     zFlashAttnQKVPackedFunc.forwardcGs4|j\}}}}}|jdkr3|jdddg|jddR}tj||j|jd} | jdd\} } } n7|jd} |jd}|jdd| |dg|jddR}tj||j|jd} | j| ||gdd\} } } t ||||||| | | |j |j |j |j |j|j| d d|jdf} | ddddddddddf S) NrFrLrGdtypedevicerrHrKr.) saved_tensorsrQrMtorchemptyrYrZrNrOrCr.r/r0r1r2rA)rRr;argsrrrrr7 qkv_shapedqkvr>r?r@rTrUr r r backwards: $  ,zFlashAttnQKVPackedFunc.backward)NNNr rr FN__name__ __module__ __qualname__ staticmethodrWrar r r r rDs ,rDc@6eZdZe        d ddZed d ZdS) FlashAttnFuncNr rr rFcCs|dur |jdd}t|||ddddddddddddddd||||f|| | | | | |d^}}}||||||||_||_| |_| |_| |_||_||_ ||fSNrrE)r/r0r1r2r4r5r6) rMr9rPr.r/r0r1r2rAr6)rRrrrr.r/r+r,r-r0r1r2r4r5rAr6rr7r8r r r rWsB zFlashAttnFunc.forwardc Gs|j\}}}}}t|t|t|}} } t||||||dddddd|| | |j|j|j|j|j|j |j |dd|j df}| dd|j df} | dd|j df} || | ddddddddddddfSN.r) r[r\ empty_likerCr.r/r0r1r2rAr6rM) rRr;r^rrrrr7r>r?r@r r r ras6""zFlashAttnFunc.backward NNNr rr rNFrrbr r r r rhs 3rhc@rg) FlashAttnVarlenFuncNr rr rFcCs| dur |jdd} t|||ddd||d|||| ddddd| | || f| ||||||d^}}}|||||||||| ||_| |_| |_| |_||_||_||_ ||_ ||_ ||fSri) rMr9rPr$r%r.r/r0r1r2rAr6)rRrrrrr r"r#r$r%r.r/r+r,r-r0r1r2r4r5rAr6rr7r8r r r rW?sNzFlashAttnVarlenFunc.forwardcGs|j\ }}}}}}} } } t|t|t|} } }t|||||||| | | |j|j| | ||j|j|j|j |j |j |j | dd|j df} | dd|j df} |dd|j df}| | |ddddddddddddddddddfSrj)r[r\rkrCr$r%r.r/r0r1r2rAr6rM)rRr;r^rrrrr7rr r"r#r>r?r@r r r ras<".zFlashAttnVarlenFunc.backwardrlrbr r r r rm=s ?rmc Cst|||||||||| | S)adropout_p should be set to 0.0 during evaluation If Q, K, V are already stacked into 1 tensor, this function will be faster than calling flash_attn_func on Q, K, V since the backward pass avoids explicit concatenation of the gradients of Q, K, V. For multi-query and grouped-query attention (MQA/GQA), please see flash_attn_kvpacked_func and flash_attn_func. If window_size != (-1, -1), implements sliding window local attention. Query at position i will only attend to keys between [i - window_size[0], i + window_size[1]] inclusive. Arguments: qkv: (batch_size, seqlen, 3, nheads, headdim) dropout_p: float. Dropout probability. softmax_scale: float. The scaling of QK^T before applying softmax. Default to 1 / sqrt(headdim). causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). window_size: (left, right). If not (-1, -1), implements sliding window local attention. softcap: float. Anything > 0 activates softcapping attention. alibi_slopes: (nheads,) or (batch_size, nheads), fp32. A bias of (-alibi_slope * |i - j|) is added to the attention score of query i and key j. deterministic: bool. Whether to use the deterministic implementation of the backward pass, which is slightly slower and uses more memory. The forward pass is always deterministic. return_attn_probs: bool. Whether to return the attention probabilities. This option is for testing only. The returned probabilities are not guaranteed to be correct (they might not have the right scaling). Return: out: (batch_size, seqlen, nheads, headdim). softmax_lse [optional, if return_attn_probs=True]: (batch_size, nheads, seqlen). The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). S_dmask [optional, if return_attn_probs=True]: (batch_size, nheads, seqlen, seqlen). The output of softmax (possibly with different scaling). It also encodes the dropout pattern (negative means that location was dropped, nonnegative means it was kept). )rDapply) rSr.r/r+r,r-r0r1r2rArTr r r flash_attn_qkvpacked_funcs-rocCs&t|||||||||| | | | | |S)a dropout_p should be set to 0.0 during evaluation Supports multi-query and grouped-query attention (MQA/GQA) by passing in KV with fewer heads than Q. Note that the number of heads in Q must be divisible by the number of heads in KV. For example, if Q has 6 heads and K, V have 2 heads, head 0, 1, 2 of Q will attention to head 0 of K, V, and head 3, 4, 5 of Q will attention to head 1 of K, V. If causal=True, the causal mask is aligned to the bottom right corner of the attention matrix. For example, if seqlen_q = 2 and seqlen_k = 5, the causal mask (1 = keep, 0 = masked out) is: 1 1 1 1 0 1 1 1 1 1 If seqlen_q = 5 and seqlen_k = 2, the causal mask is: 0 0 0 0 0 0 1 0 1 1 If the row of the mask is all zero, the output will be zero. If window_size != (-1, -1), implements sliding window local attention. Query at position i will only attend to keys between [i + seqlen_k - seqlen_q - window_size[0], i + seqlen_k - seqlen_q + window_size[1]] inclusive. Arguments: q: (batch_size, seqlen, nheads, headdim) k: (batch_size, seqlen, nheads_k, headdim) v: (batch_size, seqlen, nheads_k, headdim) dropout_p: float. Dropout probability. softmax_scale: float. The scaling of QK^T before applying softmax. Default to 1 / sqrt(headdim). causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). window_size: (left, right). If not (-1, -1), implements sliding window local attention. alibi_slopes: (nheads,) or (batch_size, nheads), fp32. A bias of (-alibi_slope * |i + seqlen_k - seqlen_q - j|) is added to the attention score of query i and key j. deterministic: bool. Whether to use the deterministic implementation of the backward pass, which is slightly slower and uses more memory. The forward pass is always deterministic. return_attn_probs: bool. Whether to return the attention probabilities. This option is for testing only. The returned probabilities are not guaranteed to be correct (they might not have the right scaling). Return: out: (batch_size, seqlen, nheads, headdim). softmax_lse [optional, if return_attn_probs=True]: (batch_size, nheads, seqlen). The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). )rhrn)rrrr.r/r+r,r-r0r1r2r4r5rAr6r r r flash_attn_funcs<rpcCs2t|||||||||| | | | | |||||||SN)rmrn)rrrrr r"r#r$r%r.r/r+r,r-r0r1r2r4r5rAr6r r r flash_attn_varlen_func(s*rrcCst||||Srq)r fwd_combine) out_partial lse_partialr out_dtyper r r flash_attn_combineTsrw cache_seqlenscache_batch_idx cache_leftpadr&rr!r$r+r,r-cCs|dksJ|ddksJd|ddksJd|dur'|jdd}|durCt|trCtj|jdf|tj|jd}t|}t |||||d| d| d|| d| || ||||||f||||||||d ^}}}|rs||g|RS|S) am If k and v are not None, k_cache and v_cache will be updated *inplace* with the new values from k and v. This is useful for incremental decoding: you can pass in the cached keys/values from the previous step, and update them with the new keys/values from the current step, and do attention with the updated cache, all in 1 kernel. If you pass in k / v, you must make sure that the cache is large enough to hold the new values. For example, the KV cache could be pre-allocated with the max sequence length, and you can use cache_seqlens to keep track of the current sequence lengths of each sequence in the batch. Also apply rotary embedding if rotary_cos and rotary_sin are passed in. The key @k will be rotated by rotary_cos and rotary_sin at indices cache_seqlens, cache_seqlens + 1, etc. If causal or local (i.e., window_size != (-1, -1)), the query @q will be rotated by rotary_cos and rotary_sin at indices cache_seqlens, cache_seqlens + 1, etc. If not causal and not local, the query @q will be rotated by rotary_cos and rotary_sin at indices cache_seqlens only (i.e. we consider all tokens in @q to be at position cache_seqlens). See tests/test_flash_attn.py::test_flash_attn_kvcache for examples of how to use this function. Supports multi-query and grouped-query attention (MQA/GQA) by passing in KV with fewer heads than Q. Note that the number of heads in Q must be divisible by the number of heads in KV. For example, if Q has 6 heads and K, V have 2 heads, head 0, 1, 2 of Q will attention to head 0 of K, V, and head 3, 4, 5 of Q will attention to head 1 of K, V. If causal=True, the causal mask is aligned to the bottom right corner of the attention matrix. For example, if seqlen_q = 2 and seqlen_k = 5, the causal mask (1 = keep, 0 = masked out) is: 1 1 1 1 0 1 1 1 1 1 If seqlen_q = 5 and seqlen_k = 2, the causal mask is: 0 0 0 0 0 0 1 0 1 1 If the row of the mask is all zero, the output will be zero. If window_size != (-1, -1), implements sliding window local attention. Query at position i will only attend to keys between [i + seqlen_k - seqlen_q - window_size[0], i + seqlen_k - seqlen_q + window_size[1]] inclusive. Note: Does not support backward pass. Arguments: q: (batch_size, seqlen, nheads, headdim) k_cache: (batch_size_cache, seqlen_cache, nheads_k, headdim) if there's no page_table, or (num_blocks, page_block_size, nheads_k, headdim) if there's a page_table (i.e. paged KV cache) page_block_size must be a multiple of 256. v_cache: (batch_size_cache, seqlen_cache, nheads_k, headdim) if there's no _table, or (num_blocks, page_block_size, nheads_k, headdim) if there's a page_table (i.e. paged KV cache) k [optional]: (batch_size, seqlen_new, nheads_k, headdim). If not None, we concatenate k with k_cache, starting at the indices specified by cache_seqlens. v [optional]: (batch_size, seqlen_new, nheads_k, headdim). Similar to k. rotary_cos [optional]: (seqlen_ro, rotary_dim / 2). If not None, we apply rotary embedding to k and q. Only applicable if k and v are passed in. rotary_dim must be divisible by 16. rotary_sin [optional]: (seqlen_ro, rotary_dim / 2). Similar to rotary_cos. cache_seqlens: int, or (batch_size,), dtype torch.int32. The sequence lengths of the KV cache. cache_batch_idx: (batch_size,), dtype torch.int32. The indices used to index into the KV cache. If None, we assume that the batch indices are [0, 1, 2, ..., batch_size - 1]. If the indices are not distinct, and k and v are provided, the values updated in the cache might come from any of the duplicate indices. cache_leftpad: (batch_size,), dtype torch.int32. The index that the KV cache starts. If None, assume 0. page_table [optional]: (batch_size, max_num_blocks_per_seq), dtype torch.int32. softmax_scale: float. The scaling of QK^T before applying softmax. Default to 1 / sqrt(headdim). causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). window_size: (left, right). If not (-1, -1), implements sliding window local attention. softcap: float. Anything > 0 activates softcapping attention. rotary_interleaved: bool. Only applicable if rotary_cos and rotary_sin are passed in. If True, rotary embedding will combine dimensions 0 & 1, 2 & 3, etc. If False, rotary embedding will combine dimensions 0 & rotary_dim / 2, 1 & rotary_dim / 2 + 1 (i.e. GPT-NeoX style). num_splits: int. If > 1, split the key/value into this many chunks along the sequence. If num_splits == 1, we don't split the key/value. If num_splits == 0, we use a heuristic to automatically determine the number of splits. Don't change this unless you know what you are doing. return_softmax_lse: bool. Whether to return the logsumexp of the attention scores. Return: out: (batch_size, seqlen, nheads, headdim). softmax_lse [optional, if return_softmax_lse=True]: (batch_size, nheads, seqlen). The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). rrrz+k_cache must have contiguous last dimensionz+v_cache must have contiguous last dimensionNrErX)r/r0r1r2r3r4r5r6) rrM isinstanceintr\fullint32rZr r9)rk_cachev_cacherrr)r*rxryrzr&rr!r$r+r,r-r.r/r0r1r2r3r4r5r6return_softmax_lserr7r8r r r flash_attn_with_kvcacheXsT qr)r rr TrNr)r rr Fr) NFNNNr rr FN) NFNNNr rr rNFr)NN)NNNNNNNNNNNNNNNFr rr TrNrF)typingrrr\torch.nnnnrr r9rCautogradFunctionrDrhrmrorprrrwr|Tensorrr r r r s  ` 7PUf > W ,