o
    TӵiW                     @   sZ   d dl Z d dlmZ d dl mZ G dd dejZG dd dejZdd	 ZdddZdS )    N)	rearrange)nnc                       s&   e Zd Zd	 fdd	Zdd Z  ZS )
LocalMHA       @   Tc                    sp   t    t|| _|| | _|| _tj||d dd| _|r*t	||d d| _
nd | _
tj||dd| _d S )N   F)bias   )
scale_base)super__init__r   	LayerNormnormheadswindow_sizeLinearto_qkvSinusoidalEmbeddingsrel_posto_out)selfdimr   dim_headuse_rotary_pos_emb	__class__ B/home/ubuntu/.local/lib/python3.10/site-packages/snac/attention.pyr      s   

zLocalMHA.__init__c                    s   |j \}}}|} |dd}| j  |jddd\}}}t fdd|||f\}}} jd urH |\}	}
t|||	|
\}}t	j
j|||}t|d} |}|dd| S )	N   r
   r   r   c                    s   t | d jdS )Nzb (w n) (h d) -> b h w n d)wh)r   r   )tr   windowsr   r   <lambda>   s    z"LocalMHA.forward.<locals>.<lambda>zb h w n d -> b (w n) (h d))shaper   	transposer   r   chunkmapr   apply_rotary_pos_embtorchr   
functionalscaled_dot_product_attentionr   r   )r   xBCTresidualqkvpos_embscaleoutr   r%   r   forward   s   
 


zLocalMHA.forward)r   r   r   T__name__
__module____qualname__r   r;   __classcell__r   r   r   r   r      s    r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )r   NFc                    s   t    ddtd|d |   }| d| || _|| _|r*|d u r*J dtd|dd|  d|  }| jd	|d
d d S )Ng      ?i'  r   r
   inv_freqz(scale base must be defined if using xposg?gffffff?r9   F)
persistent)r   r   r-   arangefloatregister_bufferuse_xposr   )r   r   r   rF   rA   r9   r   r   r   r   $   s   
zSinusoidalEmbeddings.__init__c                 C   s   |j d |j}}tj||jd| j}td|| j}tj||fdd}| js2|tj	d|dfS ||d  | j
 }| jt|d }tj||fdd}||fS )	N)devicezi , j -> i jr    r!   r   r
   zn -> n 1)r(   rH   r-   rC   type_asrA   einsumcatrF   onesr   r9   r   )r   r0   seq_lenrH   r$   freqspowerr9   r   r   r   r;   /   s   zSinusoidalEmbeddings.forward)NFr<   r   r   r   r   r   #   s    r   c                 C   s2   t | ddd} | jdd\}}tj| |fddS )Nzb ... (r d) -> b ... r dr
   )rrG   r!   r    )r   unbindr-   rK   )r0   x1x2r   r   r   rotate_half=   s   rT   r   c                 C   s   | j d }|d| d d d f }|d }|jdkr%|| d d d f }| |  | t| |  |  } ||  | t||  |  }| |fS )NrG   .r    r
   )r(   ndimcosrT   sin)r5   r6   rN   r9   q_lenq_freqs	inv_scaler   r   r   r,   C   s   

$$r,   )r   )	r-   einopsr   r   Moduler   r   rT   r,   r   r   r   r   <module>   s    