o i!@s|ddlZddlmZddlZddlmZddlmZmZddl m Z zddl m Z m Z mZmZmZWneyEd\ZZd\Z Z dZYnwz ddlmZmZmZWney`d\ZZZYnwzdd lmZWn eytdZYnwd d ZGd d d ejZGdddejZGdddejZGdddejZGdddejZddZ GdddejZ!GdddejZ"dS)N)partial) rearrangerepeat)get_dim_for_local_rank)flash_attn_kvpacked_funcflash_attn_qkvpacked_funcflash_attn_varlen_kvpacked_func flash_attn_varlen_qkvpacked_funcflash_attn_with_kvcacheNN)ColumnParallelLinear FusedDenseRowParallelLinearNNN)RotaryEmbeddingcCs\dd}t|r||Sdtt|}||td|dddd||S)Ncs6ddt|d  fddt|DS)Ncsg|]}|qSr).0iratiostartrT/home/ubuntu/.local/lib/python3.10/site-packages/xformers/_flash_attn/modules/mha.py )szCget_alibi_slopes..get_slopes_power_of_2..)mathlog2range)nheadsrrrget_slopes_power_of_2&sz/get_alibi_slopes..get_slopes_power_of_2rr)rr is_integerfloorget_alibi_slopes)rrclosest_power_of_2rrrr"%s r"cs8eZdZdZ      d fdd Zd dd ZZS) FlashSelfAttention|Implement the scaled dot product attention with softmax. Arguments --------- softmax_scale: The temperature to use for the softmax attention. (default: 1/sqrt(d_keys) where d_keys is computed at runtime) attention_dropout: The dropout rate to apply to the attention (default: 0.0) FNr(csbttdus JdtdusJd||_||_t||_|j d|dd||_ ||_ dSNzFlashAttention is not installed alibi_slopesF) persistent) super__init__r rcausal softmax_scalennDropoutdropregister_buffer window_size deterministic)selfr.r/attention_dropoutr4r*r5 __class__rrr-@   zFlashSelfAttention.__init__c Cs|jtjtjfvs J|jsJ|dur|jn|}|du}|jdur*|jtj|_|rY|jtj ks4J|dus:Jt |t sAJt ||||j rL|jjnd|j||j|j|jd St||j rb|jjnd|j||j|j|jdS)aoImplements the multihead softmax attention. Arguments --------- qkv: The tensor containing the query, key, and value. If cu_seqlens is None and max_seqlen is None, then qkv has shape (B, S, 3, H, D). If cu_seqlens is not None and max_seqlen is not None, then qkv has shape (total, 3, H, D), where total is the sum of the sequence lengths in the batch. causal: if passed, will override self.causal cu_seqlens: (batch_size + 1,), dtype torch.int32. The cumulative sequence lengths of the sequences in the batch, used to index into qkv. max_seqlen: int. Maximum sequence length in the batch. Returns: -------- out: (total, H, D) if cu_seqlens is not None and max_seqlen is not None, else (B, S, H, D). Nr&r/r.r*r4r5)dtypetorchfloat16bfloat16is_cudar.r*tofloat32int32 isinstanceintr trainingr2pr/r4r5r)r6qkvr. cu_seqlens max_seqlenunpaddedrrrforwardSs<    zFlashSelfAttention.forward)FNr&r'NFr__name__ __module__ __qualname____doc__r-rL __classcell__rrr8rr$5s r$csBeZdZdZ      d fdd Z     d dd ZZS) FlashCrossAttentionr%FNr&r'csbttdus JdtdusJd||_||_t||_|j d|dd||_ ||_ dSr)) r,r-rrr.r/r0r1r2r3r4r5)r6r.r/r7r*r4r5r8rrr-r:zFlashCrossAttention.__init__c Csh|jtjtjfvs J|jr|jsJ|dur|jn|}|du}|jdur-|jtj|_|rz|jtj ks7J|dus=Jt |t sDJ|dusJJ|jtj ksRJ|dusXJt |t s_Jt |||||||j rm|jjnd|j||j|j|jd S|jd|jd} } |jd} |jd| kr|jd|jdksJt|||j r|jjnd||j|j|j|jdS) aImplements the multihead softmax attention. Arguments --------- q: The tensor containing the query. (B, Sq, H, D) kv: The tensor containing the key and value. (B, Sk, 2, H_k, D) causal: if passed, will override self.causal cu_seqlens: (batch_size + 1,), dtype torch.int32. The cumulative sequence lengths of the sequences in the batch, used to index into q. max_seqlen: int. Maximum sequence length in the batch of q. cu_seqlens_k: (batch_size + 1,), dtype torch.int32. The cumulative sequence lengths of the sequences in the batch, used to index into kv. max_seqlen_k: int. Maximum sequence length in the batch of k and v. Nr&r;rr)r.r/r*r4r5)r<r=r>r?r@r.r*rArBrCrDrErrFr2rGr/r4r5shaper) r6qkvr.rIrJ cu_seqlens_k max_seqlen_krK batch_sizeseqlen_qseqlen_krrrrLsR     &zFlashCrossAttention.forward)FNr&Nr'F)NNNNNrMrrr8rrSs rSc,eZdZdZd fdd Zd ddZZS) SelfAttentionr%FNr&c&t||_||_t||_dSNr,r-r.r/r0r1r2r6r.r/r7r8rrr- zSelfAttention.__init__cCs|jd|jd}}|dur|jn|}|jdd\}}}|jp)dt|jd} td||| } |durQtj||fd | j | j d } | |d | t | d } |rkt tj||fd | j d d} | | j| j d} tj| d|j d} || }td||}|S)auImplements the multihead softmax attention. Arguments --------- qkv: The tensor containing the query, key, and value. (B, S, 3, H, D) causal: if passed, will override self.causal key_padding_mask: boolean mask to apply to the attention weights. True means to keep, False means to mask out. (B, S) rrTNrdim?r(bthd,bshd->bhtsr<devicer&b s -> b 1 1 srk)r<rfr<bhts,bshd->bthd)rVr.unbindr/rsqrtr=einsumfullr<rk masked_fill_rtriurAsoftmaxr2)r6rHr.key_padding_maskr[seqlenrWkvr/scores padding_mask causal_mask attentionattention_dropoutputrrrrLs(   zSelfAttention.forwardFNr&r rMrrr8rr_ r_cr^) CrossAttentionr%FNr&cr`rarbrcr8rrr-%rdzCrossAttention.__init__cCs|jd|jd}}|dur|jn|}|jd}|jd|kr*|jd|jdks,J|jd|jdkrEt|d|jd|jdd}|jdd \}} |jpYd t|jd } td ||| } |durtj ||fd | j | j d} | |d| t | d} |rt tj||j tjdd} tj||j tjd}|dur|nt |d d}|| ||k}| |d } tj| d | j d}||}td|| }|S)aImplements the multihead softmax attention. Arguments --------- q: The tensor containing the query. (B, Sq, H, D) kv: The tensor containing the key and value. (B, Sk, 2, H_k, D) causal: if passed, will override self.causal key_padding_mask: boolean mask to apply to the attention weights. True means to keep, False means to mask out. (B, Sk) rrTNrUrrz... hkv d -> ... (hkv g) d)grergr(rhrirjr&rlrkr<zs -> s 1z b -> b 1 1 1rnro)rVr.rrpr/rrqr=rrrsr<rkrtrarangelongsum masked_fillrvr2)r6rWrXr.rwr[r\r]ryrzr/r{r|row_idxcol_idxskr}r~rrrrrrL+s<  &   zCrossAttention.forwardrr rMrrr8rrrrcs.eZdZdZdejdejffdd ZZS)LinearResidualzQWrap nn.Linear to return the residual as well. For compatibility with FusedDense.inputreturncst||fSra)r,rL)r6rr8rrrL[szLinearResidual.forward)rNrOrPrQr=TensorrLrRrrr8rrXs"rc Cs|jdd\}}||jvr$tj|j|jd|||j|jd}||j|<n|j|}|j}||jd}|j }||jd} ||jdksFJ| |jdksOJ|dusUJ|||||| df<|||d| dfS)Ukv: (batch_size, seqlen, 2, nheads, head_dim) or (batch_size, 1, 2, nheads, head_dim)NrrjrrT.) rVkey_value_memory_dictr=emptymax_batch_sizerJr<rkbatch_size_offset seqlen_offset) rXinference_params layer_idx num_headshead_dimkv_cache batch_start batch_endsequence_start sequence_endrrr_update_kv_cache_s,   rcseZdZdZ                     d dfd d Zdd d ZddZddZddZ      dddZ Z S)MHA-Multi-head self-attention and cross-attentionNFTr&r@r'rc!s||d}t||_||_| |_| |_| |_| |_||_||_ ||_ |r7|s-Jdt j t ||d}nd}|dkrC|sCJd||_|durL|n||_|j|jdks[Jd|j|dksfJd |j||_|j|jd |j}d |j|j}|jdkr|rJd tdusJd t|j| |||d |_|rtdurtd|stjnt}|stnttdd}|j s|n|}|rtt||dnt}|rtt||dnt} |js|||fd|i||_n|||fd|i||_|||fd|i||_|jr*|j|jkrtj ||dd |d|_!ntj ||dd |d|_"tj ||dd |d|_#|| ||d|_$| | ||d|_%|||fd|i||_&dS)aX num_heads_kv: can be used to toggle MQA / GQA. If None, use num_heads. return_residual: whether to return the input x along with the output. This is for performance reason: for post-norm architecture, returning the input allows us to fuse the backward of nn.Linear with the residual connection. r#ALiBi code path requires flash_attnrmNr'>Local (sliding window) attention code path requires flash_attnr+num_heads must be divisible by num_heads_kv(embed_dim must be divisible by num_headsrz>MHA with rotary embedding does not support cross-attention yetrotary_emb is not installedbase scale_base interleavedrkfused_dense is not installedT)return_residualr*r4biasr) kernel_sizepaddinggroupsr.r/r7)'r,r- embed_dim cross_attnr.rdwconvrotary_emb_dimuse_flash_attnr checkpointingr=tensorr"r num_heads_kvrr rotary_embr ImportErrorr0Linearrrr$r_rSrWqkvWqWkvConv1d dwconv_qkvdwconv_q dwconv_kv inner_attninner_cross_attnout_proj)!r6rrrr qkv_proj_bias out_proj_biasdropoutr/r.rrrrotary_emb_baserotary_emb_scale_baserotary_emb_interleaved use_alibir4 fused_bias_fcrrrrkr<factory_kwargsr*qkv_dimkv_dim linear_clslinear_resid_clswqkv_clsinner_attn_clsinner_cross_attn_clsr8rrr-s          z MHA.__init__c C<|dur |jjjn|}|jjj}tj||d|j|j||dSNrrj)rweightr<rkr=rrrr6r[rJr<rkrrrallocate_inference_cache zMHA.allocate_inference_cachecCs.|jrJd|jdusJdt|||jS)rz&Generation does not support dwconv yetN0Generation requires layer_idx in the constructor)rrrr6rXrrrrrszMHA._update_kv_cachec CB|dur |jdks J|jsJ|jdkr5|jjdusJd|jj|j|j|jd|jj |jj }}nd\}}|j d}|j |j d|}|jdurT|jd|n|j}t|jdd} t||dddddf|dddddf|dddddf|dddddf||||jj|jj|jdkr|jjnd| d } | S z Fast path that combine 3 steps: apply rotary to Q and K, update kv cache, and apply attention. q: (batch_size, seqlen_q, nheads, head_dim) kv: (batch_size, seqlen_k, 2, nheads_kv, head_dim) Nrz$This code path does not support xPosrr r*rTF) rotary_cos rotary_sin cache_seqlensr/r.rotary_interleavedr*rrrrscale_update_cos_sin_cacherJrkr< _cos_cached _sin_cachedrVrrlengths_per_samplegetattrrr r/r.r r6rWrXrrrbatchrrr*contextrrr&_apply_rotary_update_kvcache_attention>     z*MHA._apply_rotary_update_kvcache_attentionc Cs|jdks tdus |js|||}|||S|jd}|j|jd|}|jdur3|jd|n|j}t |jdd}t||dddddf|dddddf|dddddf|dddddf||jj |jj |d Sz/Write kv to inference_params, then do attentionrNr*rT)rr/r.r*) rr rrrrVrrrrr/r.)r6rWrXrrrrr*rrr_update_kvcache_attention,s0     zMHA._update_kvcache_attentioncKs|dur!|dus J|dusJ|jsJ|jrJ|jdks!J|dur6|dus+J|dus1J|jr6J|durO|dus@J|durH|dusJJ|jrOJ|jrY||d|nd|i|}|duredn |jdurm|jn|j} |durw|jnd} |jdd\} } |jsH|j|j krH|dur|dusJ|j s| |} n| |\} }|jrt | t | ddddfd } t | d d |jd } |dus|jdks|jdks|jd dks|js+|jdkr|j| | | d} |dur|js|j| fi|}ndtjjj|j| fi|}nU|| dddddf| ddddddf|}n8|| dddddf| ddddddf|}n|jr|j so||durY|n|dd|f}||durk|n|}nY|dur|||\}}n||\}}||dur|n|dd|f}n2|j|j ksJ|j s| |} n| |\} }| dd|j |jf}| d|j |jdf}t |d|jd}t |dd|jd}|jrt |t |ddddfd }t |t |ddddfd }|dus |jdks |jdks |jd dks |js\|jdkr1|j||| | d\}}|durT|jsE|j||fi|}ntjjj|j||fi|}n||||}n||||}|t |d}|j sq|S||fS)a Arguments: x: (batch, seqlen, hidden_dim) (where hidden_dim = num heads * head dim) if cu_seqlens is None and max_seqlen is None, else (total, hidden_dim) where total is the is the sum of the sequence lengths in the batch. x_kv: (batch, seqlen, hidden_dim), only applicable for cross-attention. If None, use x. cu_seqlens: (batch_size + 1,), dtype torch.int32. The cumulative sequence lengths of the sequences in the batch, used to index into x. Only applicable when using FlashAttention. max_seqlen: int. Maximum sequence length in the batch. key_padding_mask: boolean mask, True means to keep, False means to mask out. (batch, seqlen). Only applicable when not using FlashAttention. mixer_subset: for cross-attention only. If not None, will take a subset of x before applying the query projection. Useful for e.g., ViT where we only care about the CLS token in the last layer. inference_params: for generation. Adapted from Megatron-LM (and Apex) https://github.com/NVIDIA/apex/blob/3ff1a10f72ec07067c4e44759442329804ac5162/apex/transformer/testing/standalone_transformer_lm.py#L470 Nr)rIrJrwrzb s d -> b d s.rzb d s -> b s dz ... (three h d) -> ... three h drthreedrrJrT... (h d) -> ... h dr ... (two hkv d) -> ... two hkv dtworz... h d -> ... (h d))rrrrrrJrVrrrrrrr contiguousrrrrr=utils checkpointrrrrrrrr)r6xx_kvrwrIrJ mixer_subsetrkwargsrrotary_max_seqlenrrxrHrrWrXoutrrrrLKs                ..$ &     z MHA.forward)NFTTr&NFNFrrNFFr'FFFFNNrNra)NNNNNN rNrOrPrQr-rrrrrLrRrrr8rr|sJ r ("rcsveZdZdZ                  d dfd d Zdd d ZddZddZddZdddZ Z S) ParallelMHArNTr&Frrr'rcs6||d}t||_| |_| |_| |_||_||_||_| |_ t j ||_||_|j|jdks:Jd|dur@|n||_|j|jdksOJdt|j|j |j|_t|j|j |j|_|j||_|j|jd|j}|r|s|Jdt|j|j }t jt||j||jd||d}nd}|d kr|sJd |jdkrtdusJd t|j| | ||d |_tdustdurtd t|||f|||j|j|jdd||_|rtt ||dnt!}|rtt"||dnt#}|| ||d|_$|| ||d|_%t|||f|||jd||_&dS)NrrrrrrrTrmr'rrrr)rsequence_parallel multiple_ofrr)'r,r-rr.rrrr process_groupsize world_sizer= distributedget_rank local_rankrrrnum_heads_per_ranknum_heads_kv_per_rankrrceilrr"rrr rrrrr$r_rSrrrr)r6rrrrrrrr/r.rrrrrrr4rrr rkr<rrnum_heads_localr*rrr8rrr-s           zParallelMHA.__init__c Crr)rrr<rkr=rrrrrrrrSrz$ParallelMHA.allocate_inference_cachecCs |jdus Jdt|||jS)rNr)rrrrrrr`szParallelMHA._update_kv_cachec Crrrrrrrrerz2ParallelMHA._apply_rotary_update_kvcache_attentionc Cs|jdks|js|||}|||S|jd}|j|jd|}|jdur/|jd|n|j}t|jdd}t ||dddddf|dddddf|dddddf|dddddf||jj |jj |d }|Sr) rrrrrVrrrrr r/r.) r6rWrXrrrrr*rrrrrs,     z%ParallelMHA._update_kvcache_attentionc Ks||}|durt|d|d}|durdn |jdur|jn|j}|dur(|jnd}|j|jkrt|dd|jd}|dusQ|jdksQ|jdksQ|jddksQ|j s|jdkr^|j |||d }|dur}|j so|j |fi|}nt jjj|j |fi|}n||dddddf|ddddd df|}n||dddddf|ddddd df|}nt|d d|j|jfd |jd } t|d |j|jdfdd|jd} |dus|jdks|jdks|jddks|j s/|jdkr|j | | ||d \} } |dur'|j s|j| | fi|}nt jjj|j| | fi|}n|| | |}n|| | |}t|d}|durEt|d}||} | S)ae Arguments: x: (batch, seqlen, hidden_dim) (where hidden_dim = num heads * head dim) if seqlen=None. If seqlen is not None, x is (batch * seqlen, hidden_dim). This is so that when we split x during sequence parallel, we split the batch * seqlen dimension (in case batch is small). Nz(b s) ... -> b s ...)srz b s (three h d) -> b s three h drrrrrT.rrrrrzb s h d -> b s (h d)zb s d -> (b s) d)rrrrrJrrrrrrrrr=rrrrrrr) r6rrxrrrHrrrrWrXr rrrrLs     ..       zParallelMHA.forward)NTTr&NFNrrNFFr'FFTNNr rar r rrr8rr s8 r (r )#r functoolsrr=torch.nnr0einopsrrflash_attn.utils.distributedr flash_attnrrrr r rflash_attn.ops.fused_denser r rflash_attn.layers.rotaryrr"Moduler$rSr_rrrrrr rrrrsD       Pa4>d