o
    i                     @   sv   d dl Z d dl mZmZ d dlmZ d dlmZ d dlmZ dd Z	G dd	 d	eZ
d
d ZeddddddZdS )    N)nneinsum)Module)autocast)	rearrangec                 C   s   | d uS )N )valr   r   J/home/ubuntu/.local/lib/python3.10/site-packages/local_attention/rotary.pyexists   s   r
   c                       s8   e Zd Z			d
 fdd	Zeddddd	 Z  ZS )SinusoidalEmbeddingsNF'  c                    s   t    d|td|d |   }| d| || _|| _|r*t|s*J dtd|dd|  d|  }| jd|d	d
 d S )Ng      ?r      inv_freqz(scale base must be defined if using xposg?gffffff?scaleF)
persistent)	super__init__torcharangefloatregister_bufferuse_xpos
scale_baser
   )selfdimr   r   thetar   r   	__class__r   r	   r      s   
zSinusoidalEmbeddings.__init__cudaenabledc                 C   s   |j d |j}}tj||jd| j}td|| j}tj||fdd}| js2|tj	d|dfS ||d  | j
 }| jt|d }tj||fdd}||fS )	N)devicezi , j -> i jr      r   zn -> n 1)shaper"   r   r   type_asr   r   catr   onesr   r   r   )r   xseq_lenr"   tfreqspowerr   r   r   r	   forward!   s   zSinusoidalEmbeddings.forward)NFr   )__name__
__module____qualname__r   r   r/   __classcell__r   r   r   r	   r      s    
r   c                 C   s2   t | ddd} | jdd\}}tj| |fddS )Nzb ... (r d) -> b ... r dr   )rr!   r$   r#   )r   unbindr   r(   )r*   x1x2r   r   r	   rotate_half2   s   r8   r   Fr   r%   c                 C   s   | j d }|d| d d d f }|d }|jdkr%|| d d d f }| |  | t| |  |  } ||  | t||  |  }| |fS )Nr!   .r#   r   )r&   ndimcosr8   sin)qkr-   r   q_lenq_freqs	inv_scaler   r   r	   apply_rotary_pos_emb7   s   

$$rA   )r%   )r   r   r   torch.nnr   	torch.ampr   einopsr   r
   r   r8   rA   r   r   r   r	   <module>   s    '
