o iG@stddlZddlmZddlmZmZmZddlZddlmZddl m Z m Z ddl m Z d dd Zd d d ZGd d d ejjZ    d!deeefdeedeefddZeZ      d"deeefdeefddZGdddejjZ    d#deeejfdeefddZGdddejjZejZ  d$deeejffddZGdddejjZdS)%N)partial)OptionalTupleUnion)Tensor) rearrangerepeat) apply_rotaryFcCsn|s|jddd\}}tj| |fddS|ddddf|ddddf}}ttj| |fdddddS)Ndim.z... d two -> ... (d two))two)chunktorchcatrstack)x interleavedx1x2rN/home/ubuntu/vllm_env/lib/python3.10/site-packages/flash_attn/layers/rotary.py rotate_halfs &rcCs|jdd}||jdksJt||sdnd}t||sdnd}tj|dd|f|t|dd|f|||d|dfgddS)z x: (batch_size, seqlen, nheads, headdim) cos, sin: (seqlen, rotary_dim / 2) or (batch_size, seqlen, rotary_dim / 2) r r z... d -> ... 1 (2 d)z... d -> ... 1 (d 2).Nr )shaperrrr)rcossinrro_dimrrrapply_rotary_emb_torchs<rc @sNeZdZe     d deeefdeedeefddZed d Z dS) ApplyRotaryEmbFrNseqlen_offsets cu_seqlens max_seqlenc Cslt||||||||d} t|tr||||||_n |||||d|_||_||_||_|s4| S|S)N)r!r"r#rinplace)r isinstanceintsave_for_backwardr!rr$r#) ctxrrrrr$r!r"r#outrrrforward's&   zApplyRotaryEmb.forwardc Cs^|j}|dur|j\}}}}n|j\}}}t||||||j|j|jdd }|dddddddfS)NT)r!r"r#rr$ conjugate)r! saved_tensorsr r#rr$)r(dor!rrr"dxrrrbackwardHs   zApplyRotaryEmb.backwardFFrNN) __name__ __module__ __qualname__ staticmethodrr&rrr*r/rrrrr &s    r r!r"r#c Ct||||||||S)aM Arguments: x: (batch_size, seqlen, nheads, headdim) if cu_seqlens is None else (total_seqlen, nheads, headdim) cos, sin: (seqlen_rotary, rotary_dim / 2) interleaved: if True, rotate pairs of even and odd dimensions (GPT-J style) instead of 1st half and 2nd half (GPT-NeoX style). inplace: if True, apply rotary embedding in-place. seqlen_offsets: (batch_size,) or int. Each sequence in x is shifted by this amount. Most commonly used in inference when we have KV cache. cu_seqlens: (batch + 1,) or None max_seqlen: int Return: out: (batch_size, seqlen, nheads, headdim) if cu_seqlens is None else (total_seqlen, nheads, headdim) rotary_dim must be <= headdim Apply rotary embedding to the first rotary_dim of x. )r apply)rrrrr$r!r"r#rrrapply_rotary_emb]sr7 num_heads_qc Cstt||||d} |dur|dur|r|dkrC|j\} } } }}| dks)J|ddddddf| | d|}| |||}n9|dksKJ| dusQJ|jd| d}|jd| d|ksgJ|ddddd| |f}| |||}|s|dkrtjt|ddd|ddddddfgdd }|Stj||dddd| |dfgdd }|S|dur|n|}|dur|n|}|dkr|j\} } } }}| dksJ|ddddd f|ddddd f}}nC|dksJ| dusJ|jd| d}|jd| d|ksJ|ddddd| f|dddd| | |f}}| |||}| |||}|sx|dkratj |||dddddfgdd }|Stj|||dddd| |dfgdd }|S) N)rr$r+r!r r zb s (t h) d -> b s t h d)tr rr) rr is_contiguousr rreshaperrrr)qkvrrcos_ksin_krr$r+r!r8apply_rotary_fnbatchseqlenthreenheadsheaddimqk num_heads_kqkrrr_apply_rotary_emb_qkvsX   &   2,  0:  &.rLc@sHeZdZe     d deeejfdeefddZ edd Z dS) ApplyRotaryEmbQKV_NFrr!r8c Csdt||||||d||d }t|tr|||||||_n ||||||d|_||_||_|S)NT)rr$r!r8)rLr%r&r'r!rr8) r(r?rrr@rArr!r8rrrr*s  zApplyRotaryEmbQKV_.forwardc Csb|j}|dur|j\}}}}}n|j\}}}}t||||||jd||jdd }|dddddddfS)NT)rr$r!r8r+)r!r,rLrr8)r(dqkvr!rrr@rArrrr/szApplyRotaryEmbQKV_.backwardNNFrN) r1r2r3r4rr&rrrr*r/rrrrrMs  rMc Cr5)a Arguments: qkv: (batch_size, seqlen, 3, nheads, headdim) or (batch_size, seqlen, num_heads_q + 2 * num_heads_k, headdim). If qkv has shape (batch_size, seqlen, num_heads_q + 2 * num_heads_k, headdim) (e.g. MQA / GQA), then num_heads_q must be provided. cos, sin: (seqlen, rotary_dim / 2) cos_k, sin_k: (seqlen, rotary_dim / 2), optional interleaved: if True, rotate pairs of even and odd dimensions (GPT-J style) instead of 1st half and 2nd half (GPT-NeoX style). seqlen_offsets: (batch_size,) or int. Each sequence in Q and K is shifted by this amount. Most commonly used in inference when we have KV cache. Return: qkv: (batch_size, seqlen, 3, nheads, headdim) or (batch_size, seqlen, num_heads_q + 2 * num_heads_k, headdim) rotary_dim must be <= headdim Apply rotary embedding *inplace* to the first rotary_dim of Q and K. )rMr6)r?rrr@rArr!r8rrrapply_rotary_emb_qkv_srPc@s6eZdZed deeejffddZeddZ dS) ApplyRotaryEmbKV_Frr!c Cs|j\}}}} } |dksJ|dddddf} t| ||||ddt|tr2|||||_n ||||d|_||_|S)Nr rT)r!rr$)rr r%r&r'r!r) r(kvrrrr!rCrDrrFrGrKrrrr* s    zApplyRotaryEmbKV_.forwardc Cs^|j}|dur|j\}}}n|j\}}t|dddddf||||jddd|ddddfS)NrT)r!rr$r+)r!r,r r)r(dkvr!rrrrrr/s  zApplyRotaryEmbKV_.backwardNFr) r1r2r3r4rr&rrr*r/rrrrrQ s rQcCst|||||S)aR Arguments: kv: (batch_size, seqlen, 2, nheads, headdim) cos, sin: (seqlen, rotary_dim / 2) interleaved: if True, rotate pairs of even and odd dimensions (GPT-J style) instead of 1st half and 2nd half (GPT-NeoX style). seqlen_offsets: (batch_size,) or int. Each sequence in Q and K is shifted by this amount. Most commonly used in inference when we have KV cache. Return: kv: (batch_size, seqlen, 2, nheads, headdim) rotary_dim must be <= headdim Apply rotary embedding *inplace* to the first rotary_dim of K. )rQr6)rRrrrr!rrrapply_rotary_emb_kv_4srUcseZdZdZ    ddeffdd Zddd Zdd d Z   dd ej de ej de eej fde ede ede ej e ej ej fff ddZ ZS)RotaryEmbeddinga5 The rotary position embeddings from RoFormer_ (Su et. al). A crucial insight from the method is that the query and keys are transformed by rotation matrices which depend on the relative positions. Other implementations are available in the Rotary Transformer repo_ and in GPT-NeoX_, GPT-NeoX was an inspiration .. _RoFormer: https://arxiv.org/abs/2104.09864 .. _repo: https://github.com/ZhuiyiTechnology/roformer .. _GPT-NeoX: https://github.com/EleutherAI/gpt-neox If scale_base is not None, this implements XPos (Sun et al., https://arxiv.org/abs/2212.10554). A recommended value for scale_base is 512: https://github.com/HazyResearch/flash-attention/issues/96 Reference: https://github.com/sunyt32/torchscale/blob/main/torchscale/component/xpos_relative_position.py @FNr cst||_t||_||}|jd|dd||_||_|dur7t j d|d|t j dd|d |nd}|jd |ddd|_ d|_ d|_d|_d|_dS) z interleaved: if True, rotate pairs of even and odd dimensions (GPT-J style) instead of 1st half and 2nd half (GPT-NeoX style). inv_freqF) persistentNrr devicedtypeg?gffffff?scale)super__init__r floatbase_compute_inv_freqregister_bufferr scale_baserarangefloat32_seq_len_cached _cos_cached _sin_cached _cos_k_cached _sin_k_cached)selfr rarrdr[rXr] __class__rrr_]s"   & zRotaryEmbedding.__init__c Cs(d|jtjd|jd|tjd|jS)Ng?rr rZ)rarrer rf)rlr[rrrrb~s z!RotaryEmbedding._compute_inv_freqc CsX||jks|jdus|jj|ks|jj|ks|jr|jr||_tj||tjd}|j jtjkr8|j |d}n|j }t ||}|j durZt |||_t|||_dStj||j j|j jd|d|j}|j j|jdt|d}t ||||_t||||_t ||||_t||||_dSdSdS)NrZ)r[)r\r[r zs -> s 1)rgrhr[r\training is_inferencerrerfrXrbouterr]rtorrirdrrjrk) rlrDr[r\r<rXfreqspowerr]rrr_update_cos_sin_caches8      z%RotaryEmbedding._update_cos_sin_cacherr?rR seqlen_offsetr#r8returnc Cs|jd}|dur|j||j|jdnt|tr%|j|||j|jd|durHt||j|j|j dur7|j nd|j dur@|j nd|j ||dS|}t ||j|j|j d|d}t||j dura|jn|j |j durk|jn|j |j |d}||fS)a* qkv: (batch, seqlen, 3, nheads, headdim) or (batch, seqlen, num_heads_q + 2 * num_heads_k, headdim) if kv is none, else it's just q of shape (batch, seqlen, nheads, headdim). If qkv has shape (batch, seqlen, num_heads_q + 2 * num_heads_k, headdim) (e.g. MQA / GQA), then num_heads_q must be provided. kv: (batch, seqlen, 2, nheads, headdim) seqlen_offset: (batch_size,) or int. Each sequence in x is shifted by this amount. Most commonly used in inference when we have KV cache. If it's a tensor of shape (batch_size,), then to update the cos / sin cache, one should pass in max_seqlen, which will update the cos / sin cache up to that length. Apply rotary embedding *inplace* to qkv and / or kv. rNrZ)rr!r8T)rr$r!)rr!)rrur[r\r%r&rPrhrir]rjrkrapply_rotary_emb_funcrU)rlr?rRrvr#r8rDrJrrrr*sB   zRotaryEmbedding.forward)rWFNN)N)NN)NrNN)r1r2r3__doc__r&r_rbrurrrrrr* __classcell__rrrmrrVKs8 ! , rV)Fr0)NNFFFrNrOrT)math functoolsrtypingrrrrreinopsrrflash_attn.ops.triton.rotaryr rrautogradFunctionr r&r7rxrLrMrPrQr6rUnnModulerVrrrrsl     ;  "  @.  &