o iE8@sddlZddlmZddlZddlmZmZddlmZm Z dddZ  dd d Z    dd d Z    dddZ              ddeejfddZdS)N)Optional) rearrangerepeat) pad_input unpad_inputrandomFcCs|dvsJ|dkrtj|df||tjd}n.|dkr2tjt|r"dnd|d|d|df|d}n|d krEtj|d |d|df|d}|r\t|D] }|d dkrWd||<qKd|d <ttj||dd |d|k}|S)N)fullrthirdrdevicedtyperr)r r zs -> b sb)torchrint32randintmaxrangerarange) max_seqlen batch_sizer mode zero_lengthslengthsi padding_maskr!N/home/ubuntu/vllm_env/lib/python3.10/site-packages/flash_attn/utils/testing.pygenerate_random_padding_mask s$ "  r#c #sN|r|rJ|j\} } |jd} |j\} }} |j|| fks$J|j|| fks/J|dus7| dur?|r;J|r?J|durdt|||\}}}}fdd}|durat|dnd}n*t|d}tjddtj|jd }d}}fd d}|durt|dnd}|durt||| \}}}}t||| ^}}nt|d}t|d}tjddtj|jd }d}}|r ||ksJ| |ksJtj|||gdd }tj|||gd d }|durfd d}nfdd}| ||| ||fS|rTtj||gdd }tj||gd d }|} |dur1fdd}!nfdd}!| | ||||| | || |!f S|} |durdfdd}"nfdd}"| | | |dur| nd||||||| | | |dur| nd|| |"fS)a  Arguments: q: (batch_size, seqlen_q, nheads, d) k: (batch_size, seqlen_k, nheads_k, d) v: (batch_size, seqlen_k, nheads_k, d_v) query_padding_mask: (batch_size, seqlen), bool key_padding_mask: (batch_size, seqlen), bool rNct|SNr output_unpadr indices_qseqlen_qr!r"<szgenerate_qkv..zb s ... -> (b s) ...zb s h d -> (b s) h drr )stepr r ct|ddSNz(b s) h d -> b s h drrr'rr!r"r,Gdimcr$r%r& dqkv_unpadr)r!r"r,`cr.Nz(b s) t h d -> b s t h drr0r6r1r!r"r,br2cr$r%r& dkv_unpadr indices_kseqlen_kr!r"r,rr8cr.r9r0r:r1r!r"r,tr2cr$r%r&dk_unpadr<r!r"r,r8cr.r/r0r?r1r!r"r,r8) shaperrrrrr allstackdetachrequires_grad_)#qkvquery_padding_maskkey_padding_maskqvkvpacked qkvpackedquery_unused_maskkey_unused_masknheadsdd_v_nheads_kq_unpad cu_seqlens_q max_seqlen_q seqused_q output_pad_fnqv_unpadk_unpad cu_seqlens_k max_seqlen_k seqused_kv_unpadrest qkv_unpadqkv dqkv_pad_fnkv_unpadkv dq_pad_fn dkv_pad_fn dk_pad_fnr!)rr=r*r>r+r" generate_qkv"s                        riNNc Cs"ttj||tjdd}tj||tjd} |dur3t|d}t| d|jdd} t| |k| |d} |dur9|nt|dd} |durG|nt|dd} |ddura| || | |d kS|durkt| |n| } t | t || | |d | kt | || | |dk| |kS) Nr s -> s 1 b -> b 1 1 1 s -> b 1 1 srrrr ) rrrlongrrAwheresum full_like logical_orminimum logical_and) r+r> window_sizesink_token_lengthrIrJ key_leftpadr row_idxcol_idxsksqr!r!r"construct_local_masks*   "r}c Csttj||tjdd}tj||tjd}|dur3t|d}t|d|jdd}t||k||d}|dur9|nt|dd} |durG|nt|dd} |durYt||n| } || | || | |} t || k|| |kS) Nr rkrlrmrrrnr) rrrrorrArprqrrrs) r+r>attention_chunkrIrJrxr ryrzr{r|col_limit_left_chunkr!r!r"construct_chunk_masks&  rTlearnable_sinkc( Cs| r|ddf}|j}|r%|||}}}| dur#| nd} | durRt| d|jd|jdd} || |j}| durP| | | jnd} | durd|t| dj|jd}| durv|t| dj|jd}|jd|jd}}t|d |jd|jdd}t|d |jd|jdd}|jd }|jd }d t| dur|n||}|st d |||}n t d |||}| dur|t d | ||}|dkrt |||}|dur| t|d tdd}|ddus |ddurt ||||||||j d}|dkr4t|||||||j d}|dur2t||n|}|durA| |td|durJ||}|dur[tj|d d|j}n6|tj} tj| d dd}!t|d}t||!}"t| |"}#|#jd ddt||"}$|#|$|j}|dur|t|dd}|dur|t|d d}|dur|tj|d ddd}d d|}%|dur||d}&n|}&|dur|&||&j}&t d|&||%}'|dur|' t|dd|'j|d|j|dfS)a Arguments: q: (batch_size, seqlen_q, nheads, head_dim) k: (batch_size, seqlen_k, nheads, head_dim) v: (batch_size, seqlen_k, nheads, head_dim_v) qv: (batch_size, seqlen_q, nheads, head_dim_v) query_padding_mask: (batch_size, seqlen_q) key_padding_mask: (batch_size, seqlen_k) attn_bias: broadcastable to (batch_size, nheads, seqlen_q, seqlen_k) dropout_p: float dropout_mask: (batch_size, nheads, seqlen_q, seqlen_k) causal: whether to apply causal masking upcast: whether to cast all inputs to fp32, do all computation in fp32, then cast output back to fp16/bf16. reorder_ops: whether to change the order of operations (scaling k instead of scaling k, etc.) without changing the math. This is to estimate the numerical error from operation reordering. Output: output: (batch_size, seqlen_q, nheads, head_dim_v) attention: (batch_size, nheads, seqlen_q, seqlen_k), softmax after dropout rNzb h -> b 1 (h g) 1r5)gzb h -> b 1 h 1)r r zb s h d -> b s (h g) drg?zbthd,bshd->bhtszb s -> b 1 1 sz-inf)rxr r3T)r4keepdimz h -> h 1 1zb s -> b 1 s 1rzbhts,bshd->bthdzb s -> b s 1 1)r floatrrAtormathsqrtreinsumtanh masked_fill_r}r rrssoftmaxfloat32amaxmaximumexprq masked_fillrB)(rFrGrHrIrJrx attn_bias dropout_p dropout_maskcausalrK q_descale k_descale v_descalervr~rwrsoftcapupcast reorder_opsintermediate_dtypedtype_ogr+r>rQdv softmax_scalescores local_mask chunk_mask attention scores_fp32 logits_maxlogits_or_sinks_maxunnormalized_scores normalizerdropout_scalingattention_dropoutputr!r!r" attention_refs+                  r)rF)NNNFFNN)rjrNNNN)NNNN)NNNNrNFNNNNrjrrNrTFN)rtypingrreinopsrrflash_attn.bert_paddingrrr#rir}rTensorrr!r!r!r"sT    ( %