o ۷iSL@sddlmZmZmZmZddlZddlmZddlZ ej j Z ddZ ddZ dedefd d Zejjd d d d                              dgdejdejdejdeejdeejdeejdeejdeejdeejdeejdeejdeejd eed!eed"eejd#eejd$eejd%eejd&eejd'eejd(eejd)eejd*eejd+eed,ed-ed.ed/ed0ed1ed2eejd3ed4eed5edeejejejejffFd6d7Zejd                               dgdejdejdejdeejdeejdeejdeejdeejdeejdeejdeejdeejd eed!eed"eejd#eejd$eejd%eejd&eejd'eejd(eejd)eejd*eejd+eed,ed-ed.ed/ed0ed1ed2eejd3ed4eed5edeejejejejffFd8d9Zejjd:d;d d                dhdejdeejdeejd?eejd@eejd eed!eedAeejdBeejdCeejd+eedDed-ed.ed0edEed5edejf.dFdGZejd:                dhdejdeejdeejd?eejd@eejd eed!eedAeejdBeejdCeejd+eedDed-ed.ed0edEed5edejf.dHdIZdJdKZdLdMZejeedNGdOdPdPejjZGdQdRdRejjZ GdSdTdTejjZ!    U    didVdWZ"     U     djdXdYZ#       U     dkdZd[Z$dld\d]Z%                   U     dmd^eeeejfd_eejd`eejd"eejdeejdeejd eedaeejd(eejd)eejd*eejfdbdcZ&ej'ddddddddUdddddfd^ejdeejdeejd`eejddeef dedfZ(dS)n)OptionalUnionListTupleNcCs"|dur|ddkr|S|S)N)stride contiguous)xr G/home/ubuntu/vllm_env/lib/python3.10/site-packages/fa3_fwd_interface.pymaybe_contiguouss"r cCs||d||S)Nrr )r mr r r round_multiplesr head_sizereturncCsddlm}|dds|dkrdS|dds|dkrdS|dds*|d kr*d S|dd s6|d kr6d S|dd sB|d krBd Sd S)Nr)CONFIG build_flagsFLASHATTENTION_DISABLE_HDIM64@FLASHATTENTION_DISABLE_HDIM96`FLASHATTENTION_DISABLE_HDIM128FLASHATTENTION_DISABLE_HDIM192FLASHATTENTION_DISABLE_HDIM256)flash_attn_configr)rrr r r round_up_headdims"      rzfa3_fwd::_flash_attn_forwardr cuda) mutates_args device_typesFrTrqkvk_newv_newqvout_ cu_seqlens_q cu_seqlens_kcu_seqlens_k_new seqused_q seqused_k max_seqlen_q max_seqlen_k page_table kv_batch_idx leftpad_k rotary_cos rotary_sinseqlens_rotary q_descale k_descale v_descale softmax_scalecausalwindow_size_leftwindow_size_rightattention_chunksoftcaprotary_interleavedscheduler_metadata num_splitspack_gqa sm_marginc"&Csdd||||fD\}}}}|ddkr!|ddkr!|n|}dd||| fD\}}} dd| | fD\} } dd|||fD\}}}d d||fD\}}t|}tjg|||||||||| | | | | ||||||||||||||||||| |!R\}"}#}$}%|$durtjg|"jd }$|%durtjg|"jd }%|"|#|$|%fS) NcSg|]}t|qSr r .0r r r r Rz'_flash_attn_forward..rrcSrFr rGrHr r r rJTcSrFr rGrHr r r rJWrKcSrFr rGrHr r r rJXrMcSrFr rGrHr r r rJ[rKdevice)rr r fa3_fwd_cudafwdtorchtensorrO)&r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrEout softmax_lse out_accumsoftmax_lse_accumr r r _flash_attn_forward-s%(        !"% rXc"/Cs|du}"|"r|j\}#}$}%|jdd}&| durtd| }'n|j\}&}'}$}%|&|jd}#|jd}(|j})|)tjkr=tj}*n|)}*|durGtd|"rVtj|#|$|(f|*|jd}+n tj|&|'|$|(f|*|jd}+|"rrtj|$|#ftj |jd},n tj|&|$|'ftj |jd},|dkrtd||dkr|"rtj||$|#|(ftj |jd}-tj||$|#ftj |jd}.n.tj||&|$|'|(ftj |jd}-tj||&|$|'ftj |jd}.ntj g|+jd }-tj g|+jd }.|+|,|-|.fS) z Symbolic fake implementation of flash attention forward. Returns tensors with the correct shapes and dtypes without actual computation. Nrrz9max_seqlen_q must be provided if cu_seqlens_q is providedrzWTracing (torch.compile/torch.export) with pre-allocated output tensor is not supported.dtyperOzXtracing (torch.compile/torch.export) with num_splits <= 0 not supported. Got num_splits=rN) shape ValueErrorrZrR float8_e4m3fnbfloat16 TypeErroremptyrOfloat32rS)/r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrE is_varlen_qtotal_q num_headsr batch_sizeseqlen_q head_size_vq_type out_dtyperTrUrVrWr r r _flash_attn_forward_fakesB+    rjzfa3_fwd::_flash_attn_backward)dqdkdvdoutrTrUsequed_qsequed_krkrlrm is_causal deterministiccCs^dd|||||fD\}}}}}t||||||| | ||||| | | |||||||^}}|S)NcSrFr rGrHr r r rJrKz(_flash_attn_backward..)rPbwd)rnr$r%r&rTrUr+r,rorpr0r1rkrlrmr;rqr=r>r@rrrE softmax_drestr r r _flash_attn_backwards4"rvc+CsP|du}|du}|p|p|dup| du}|s-|d}|d}|d}||d}n|dd}|d}| }| }||dkrEd}||dkrMd}|rQd}|dkoX|dk}|d}|d}tt||}tj|j} | dd| d}!|dks|dko| }"|dkr|r|dkrdnd}#n|dkrd}#n|dkr|s|"s|dkrdnd }#nd}#|dkrdnd}$|d krdnd }%|!d kr|#}&n |!d ks|!dkr|%}&n|$}&|jd}'t||&}(t|||&|&})| durt |n| } | durt |n| } |durt |n|}|stj ||'|(ftj |jd}*|*Stj |'|)ftj |jd}*|*S)Nrrr rr#rrPr ZVYrY) sizermaxrRr get_device_capabilityrOr[r empty_liker`ra)+rnr$r%r&rTrUr+r,rorpr0r1rkrlrmr;rqr=r>r@rrrErb is_varlen_k is_varlenrerfseqlen_krcrrghead_size_roundedcaparchis_local kBlockM_sm90 kBlockM_sm80 kBlockM_sm86kBlockMrdseqlen_q_roundedtotal_q_padded_roundedrtr r r _flash_attn_backward_fake*sd          rc Csx|dd\}}}|\}}}}|||||||d|_|d|_|d|dg|_|d|_|d|_|d|_dS) Niiiiiir)save_for_backwardr;r< window_sizer?r@rE) ctxinputsoutputr$r%r&rTrU_r r r setup_contexts     rc Gs|j\}}}}}t|t|t|}} } t||||||dddddd|| | |j|j|jd|jd|jd|j|| | gdRS)NrrF)NNNNNNNNNNNNNNNNNNNNN) saved_tensorsrRrrvr;r<rr@rE) rrngradsr$r%r&rTrUrkrlrmr r r _backwards0"r)rc@s6eZdZe        d ddZedd ZdS) FlashAttnQKVPackedFuncNrrrr#FcCsV|dur |jdd}|dkr$|jddksJ|jdd\}}}n1|dks,J| dus2J|jd| d}|d| |jdksHJ|j| ||gd d\}}}t|||ddddddddddddddddd||||f||d |d || | d ^}}}||||||||_||_||_||_ | |_ | |_ ||_ | |_ | r||fS|S) NrrLrdimr}rr)r<r=r>r?r@rE)r[runbindsplitrXrr;r<rr?r@rrndimrE)rqkvr;r<r8r9r:rr?r@rr num_heads_qrEreturn_softmaxr$r%r& num_heads_krTrUrur r r forwardsT   zFlashAttnQKVPackedFunc.forwardcGsb|j\}}}}}|jdksJd|jdkr<|jdddg|jddR}tj||j|jd} | jdd\} } } n7|jd } |jd }|jdd| |d g|jd dR}tj||j|jd} | j | ||gdd\} } } t ||||||dddddd| | | |j |j |j d|j d |j|j|j| d d|jd f} | ddddddddddddf S) Nr-FA3 backward does not support attention_chunkrr}rrYrLrrrr.)rr?rr[rRr`rZrOrrrvr;r<rr@rrrE)rrnargsr$r%r&rTrU qkv_shapedqkvrkrlrmrrr r r backwardsD $  ,zFlashAttnQKVPackedFunc.backward) NNNrrr#FNrF__name__ __module__ __qualname__ staticmethodrrr r r r rs :rc@:eZdZe          d ddZed d ZdS) FlashAttnFuncNrrr#rFcCs|dur|jd|dur|jdndd}t|||dd|dddddddddddddd||| |f|| d| d| | | ||d^}}}||||||||_||_| |_| |_| |_||_||_ |rg||fS|SNrrrr)r<r=r>r?r@rCrDrE) r[rXrr;r<rr?r@rrrE)rr$r%r&r;r<r)r8r9r:rr?r@rCrDrrrErrTrUrur r r rsF$zFlashAttnFunc.forwardc Gs|j\}}}}}|jdksJdt|t|t|}} } t||||||dddddd|| | |j|j|jd|jd|j|j |j |dd|j df}| dd|j df} | dd|j df} || | ddddddddddddddfSNrrr.r) rr?rRrrvr;r<rr@rrrEr[) rrnrr$r%r&rTrUrkrlrmr r r rTs8"&zFlashAttnFunc.backward NNNNrrr#rNFrFrr r r r rs 7rc@r) FlashAttnVarlenFuncNrrr#rFcCs| dur|jd| dur| jdndd} t|||dd| d||d|||| dddddd| ||| f| |d|d|||||d^}}}|||||||||| ||_| |_| |_| |_||_||_||_ ||_ ||_ |rq||fS|Sr) r[rXrr0r1r;r<rr?r@rrrE)rr$r%r&r+r,r.r/r0r1r;r<r)r8r9r:rr?r@rCrDrrrErrTrUrur r r rvsR$zFlashAttnVarlenFunc.forwardcGs |j\ }}}}}}} } } |jdksJdt|t|t|} } }t|||||||| | | |j|j| | ||j|j|j d|j d|j |j |j | dd|j df} | dd|j df} |dd|j df}| | |ddddddddddddddddddddfSr)rr?rRrrvr0r1r;r<rr@rrrEr[)rrnrr$r%r&rTrUr+r,r.r/rkrlrmr r r rs>"2zFlashAttnVarlenFunc.backwardrrr r r r rts Crrc Cs"t|||||||||| | | | S)adropout_p should be set to 0.0 during evaluation If Q, K, V are already stacked into 1 tensor, this function will be faster than calling flash_attn_func on Q, K, V since the backward pass avoids explicit concatenation of the gradients of Q, K, V. For multi-query and grouped-query attention (MQA/GQA), please see flash_attn_kvpacked_func and flash_attn_func. If window_size != (-1, -1), implements sliding window local attention. Query at position i will only attend to keys between [i - window_size[0], i + window_size[1]] inclusive. Arguments: qkv: (batch_size, seqlen, 3, nheads, headdim) dropout_p: float. Dropout probability. softmax_scale: float. The scaling of QK^T before applying softmax. Default to 1 / sqrt(headdim). causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). window_size: (left, right). If not (-1, -1), implements sliding window local attention. softcap: float. Anything > 0 activates softcapping attention. alibi_slopes: (nheads,) or (batch_size, nheads), fp32. A bias of (-alibi_slope * |i - j|) is added to the attention score of query i and key j. deterministic: bool. Whether to use the deterministic implementation of the backward pass, which is slightly slower and uses more memory. The forward pass is always deterministic. return_attn_probs: bool. Whether to return the attention probabilities. This option is for testing only. The returned probabilities are not guaranteed to be correct (they might not have the right scaling). Return: out: (batch_size, seqlen, nheads, headdim). softmax_lse [optional, if return_attn_probs=True]: (batch_size, nheads, seqlen). The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). S_dmask [optional, if return_attn_probs=True]: (batch_size, nheads, seqlen, seqlen). The output of softmax (possibly with different scaling). It also encodes the dropout pattern (negative means that location was dropped, nonnegative means it was kept). )rapply) rr;r<r8r9r:rr?r@rrrrEreturn_attn_probsr r r flash_attn_qkvpacked_funcs/rcCs*t|||||||||| | | | | |||S)a dropout_p should be set to 0.0 during evaluation Supports multi-query and grouped-query attention (MQA/GQA) by passing in KV with fewer heads than Q. Note that the number of heads in Q must be divisible by the number of heads in KV. For example, if Q has 6 heads and K, V have 2 heads, head 0, 1, 2 of Q will attention to head 0 of K, V, and head 3, 4, 5 of Q will attention to head 1 of K, V. If causal=True, the causal mask is aligned to the bottom right corner of the attention matrix. For example, if seqlen_q = 2 and seqlen_k = 5, the causal mask (1 = keep, 0 = masked out) is: 1 1 1 1 0 1 1 1 1 1 If seqlen_q = 5 and seqlen_k = 2, the causal mask is: 0 0 0 0 0 0 1 0 1 1 If the row of the mask is all zero, the output will be zero. If window_size != (-1, -1), implements sliding window local attention. Query at position i will only attend to keys between [i + seqlen_k - seqlen_q - window_size[0], i + seqlen_k - seqlen_q + window_size[1]] inclusive. Arguments: q: (batch_size, seqlen, nheads, headdim) k: (batch_size, seqlen, nheads_k, headdim) v: (batch_size, seqlen, nheads_k, headdim) dropout_p: float. Dropout probability. softmax_scale: float. The scaling of QK^T before applying softmax. Default to 1 / sqrt(headdim). causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). window_size: (left, right). If not (-1, -1), implements sliding window local attention. alibi_slopes: (nheads,) or (batch_size, nheads), fp32. A bias of (-alibi_slope * |i + seqlen_k - seqlen_q - j|) is added to the attention score of query i and key j. deterministic: bool. Whether to use the deterministic implementation of the backward pass, which is slightly slower and uses more memory. The forward pass is always deterministic. return_attn_probs: bool. Whether to return the attention probabilities. This option is for testing only. The returned probabilities are not guaranteed to be correct (they might not have the right scaling). Return: out: (batch_size, seqlen, nheads, headdim). softmax_lse [optional, if return_attn_probs=True]: (batch_size, nheads, seqlen). The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). )rr)r$r%r&r;r<r)r8r9r:rr?r@rCrDrrrErr r r flash_attn_funcs">rcCs6t|||||||||| | | | | |||||||||SN)rr)r$r%r&r+r,r0r1r.r/r;r<r)r8r9r:rr?r@rCrDrrrErr r r flash_attn_varlen_funcls.rcCst||||Sr)rP fwd_combine) out_partial lse_partialrTrir r r flash_attn_combinesr cache_seqlenscache_batch_idx cache_leftpadrotary_seqlensc!Cs|ddks Jd|ddksJd|dur,|jd|dur'|jdndd}|durHt|trHtj|jdf|tj|jd}t|}t ||||||d| d| d||d| | | |||||||f||d|d|||||||d ^}}} |r||g| RS|S) a If k and v are not None, k_cache and v_cache will be updated *inplace* with the new values from k and v. This is useful for incremental decoding: you can pass in the cached keys/values from the previous step, and update them with the new keys/values from the current step, and do attention with the updated cache, all in 1 kernel. If you pass in k / v, you must make sure that the cache is large enough to hold the new values. For example, the KV cache could be pre-allocated with the max sequence length, and you can use cache_seqlens to keep track of the current sequence lengths of each sequence in the batch. Also apply rotary embedding if rotary_cos and rotary_sin are passed in. The key @k will be rotated by rotary_cos and rotary_sin at indices cache_seqlens, cache_seqlens + 1, etc. If causal or local (i.e., window_size != (-1, -1)), the query @q will be rotated by rotary_cos and rotary_sin at indices cache_seqlens, cache_seqlens + 1, etc. If not causal and not local, the query @q will be rotated by rotary_cos and rotary_sin at indices cache_seqlens only (i.e. we consider all tokens in @q to be at position cache_seqlens). See tests/test_flash_attn.py::test_flash_attn_kvcache for examples of how to use this function. Supports multi-query and grouped-query attention (MQA/GQA) by passing in KV with fewer heads than Q. Note that the number of heads in Q must be divisible by the number of heads in KV. For example, if Q has 6 heads and K, V have 2 heads, head 0, 1, 2 of Q will attention to head 0 of K, V, and head 3, 4, 5 of Q will attention to head 1 of K, V. If causal=True, the causal mask is aligned to the bottom right corner of the attention matrix. For example, if seqlen_q = 2 and seqlen_k = 5, the causal mask (1 = keep, 0 = masked out) is: 1 1 1 1 0 1 1 1 1 1 If seqlen_q = 5 and seqlen_k = 2, the causal mask is: 0 0 0 0 0 0 1 0 1 1 If the row of the mask is all zero, the output will be zero. If window_size != (-1, -1), implements sliding window local attention. Query at position i will only attend to keys between [i + seqlen_k - seqlen_q - window_size[0], i + seqlen_k - seqlen_q + window_size[1]] inclusive. Note: Does not support backward pass. Arguments: q: (batch_size, seqlen, nheads, headdim) k_cache: (batch_size_cache, seqlen_cache, nheads_k, headdim) if there's no page_table, or (num_blocks, page_block_size, nheads_k, headdim) if there's a page_table (i.e. paged KV cache) page_block_size can be arbitrary (e.g, 1, 2, 3, 64, etc.). v_cache: (batch_size_cache, seqlen_cache, nheads_k, headdim_v) if there's no page_table, or (num_blocks, page_block_size, nheads_k, headdim_v) if there's a page_table (i.e. paged KV cache) k [optional]: (batch_size, seqlen_new, nheads_k, headdim). If not None, we concatenate k with k_cache, starting at the indices specified by cache_seqlens. v [optional]: (batch_size, seqlen_new, nheads_k, headdim_v). Similar to k. qv [optional]: (batch_size, seqlen, nheads, headdim_v) rotary_cos [optional]: (seqlen_ro, rotary_dim / 2). If not None, we apply rotary embedding to k and q. Only applicable if k and v are passed in. rotary_dim must be divisible by 16. rotary_sin [optional]: (seqlen_ro, rotary_dim / 2). Similar to rotary_cos. cache_seqlens: int, or (batch_size,), dtype torch.int32. The sequence lengths of the KV cache. cache_batch_idx: (batch_size,), dtype torch.int32. The indices used to index into the KV cache. If None, we assume that the batch indices are [0, 1, 2, ..., batch_size - 1]. If the indices are not distinct, and k and v are provided, the values updated in the cache might come from any of the duplicate indices. cache_leftpad: (batch_size,), dtype torch.int32. The index that the KV cache starts. If None, assume 0. page_table [optional]: (batch_size, max_num_blocks_per_seq), dtype torch.int32. softmax_scale: float. The scaling of QK^T before applying softmax. Default to 1 / sqrt(headdim). causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). window_size: (left, right). If not (-1, -1), implements sliding window local attention. softcap: float. Anything > 0 activates softcapping attention. rotary_interleaved: bool. Only applicable if rotary_cos and rotary_sin are passed in. If True, rotary embedding will combine dimensions 0 & 1, 2 & 3, etc. If False, rotary embedding will combine dimensions 0 & rotary_dim / 2, 1 & rotary_dim / 2 + 1 (i.e. GPT-NeoX style). num_splits: int. If > 1, split the key/value into this many chunks along the sequence. If num_splits == 1, we don't split the key/value. If num_splits == 0, we use a heuristic to automatically determine the number of splits. Don't change this unless you know what you are doing. return_softmax_lse: bool. Whether to return the logsumexp of the attention scores. Return: out: (batch_size, seqlen, nheads, headdim). softmax_lse [optional, if return_softmax_lse=True]: (batch_size, nheads, seqlen). The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). rrz+k_cache must have contiguous last dimensionz+v_cache must have contiguous last dimensionNrrrY) r<r=r>r?r@rArBrCrDrE) rr[ isinstanceintrRfullint32rOr rX)!r$k_cachev_cacher%r&r)r5r6rrrr2r+r-r0rr8r9r:r;r<rr?r@rArBrCrDrEreturn_softmax_lserTrUrur r r flash_attn_with_kvcachesZu$#r page_sizecCsXt|}|dur |}t|||||||||| d| d| | | ||d|d|||||}|S)Nrr)r rPget_scheduler_metadata)rer0r1r num_heads_kvheaddimr qkv_dtype headdim_vr+r-rrmax_seqlen_k_newr<rr? has_softcaprCrDrErBr r r rDs. r)NNNNNNNNNNNNNNNNNNNNNFrrrr#TNrNr)NNNNNNNNNNFrrr#Fr) NFNNNrrr#FNrF)NFNNNNrrr#rNFrF)NNNFNNNNrrr#rNFrF)NN)NNNNNNNNNNNNNNNNNFrrr#TNrNrF))typingrrrrrRtorch.nnnn fa3_fwd._Cfa3_fwdopsrPr rrrlibrary custom_opTensorfloatboolrX register_fakerjrvrrrregister_autogradautogradFunctionrrrrrrrrr^rr r r r s       !"# ]      !"# h      5      a cZk B Y 0      (