o
    ίi)                     @   s   d dl mZmZ d dlmZmZmZ d dlZd dlm	Z	m
Z
 d dlmZmZmZmZ d dlmZ d dlmZmZ dd	 Zd
d ZdddZdd ZeddddddZd ddZG dd deZG dd dZdS )!    )logpi)LiteralOptionalUnionN)	rearrangerepeat)Tensorbroadcast_tensorseinsumnn)autocast)Module
ModuleListc                 C   s   | d uS N )valr   r   L/home/ubuntu/.local/lib/python3.10/site-packages/core/vision_encoder/rope.pyexists   s   r   c                 C   s   t | r| S |S r   )r   )r   dr   r   r   default   s   r   c                 C   s   t |  }tj||dS )Ndim)r
   torchcat)tensorsr   broadcasted_tensorsr   r   r   broadcat   s   r   c                 C   s<   t | ddd} | jdd\}}tj| |fdd} t | dS )Nz... (d r) -> ... d r   rr   r   z... d r -> ... (d r))r   unbindr   stack)xx1x2r   r   r   rotate_half    s   
r'   cudaFenabled      ?c                 C   s   |j }|jdkr|j| }| | d  } | jd }|| }||jd ks1J d|jd  d| |dd |f |d||f |d|d f }	}}
||   | t||   |  }tj|	||
fdd}||S )N   r   zfeature dimension z: is not of sufficient size to rotate in all the positions .r   )	dtypendimshapecosr'   sinr   r   type)freqststart_indexscaleseq_dimr.   seq_lenrot_dim	end_indext_leftt_rightoutr   r   r   apply_rotary_emb'   s    



$
r?   c                 C   s:   t |rtd| |} t| d} t| ddd} t| ||dS )N..., f -> ... fz... r f -> ... (r f)... n -> ... (n r)r   r    )r6   )r   r   r   r   r?   )	rotationsr5   r6   freq_rangesr   r   r   apply_learned_rotationsD   s
   
rD   c                	       s   e Zd Z													d'd
ee deed ed ed f f fddZedd Z	dd Z
d(ddZd)ddZd)ddZd*ddZd)dedee fdd Zd!d" Zed#dd$d)defd%d&Z  ZS )+RotaryEmbeddingNlang'  
      F   r+   Tcustom_freqs	freqs_forpixelconstantc                    sd  t    ||||d   9 }|| _t|r|}n7|dkr3d|td|dd |d   |   }n|dkrEtd|d |d t }n|dkrPt	| }|| _
| dd  | dd  tj||d	| _|| _| d
td || _|rzdnd| _|
dksJ |
| _|| _|s| dd  d S td|dd|  d|  }|	| _| d| tt| _d S )Nr   rF   r+   r   rM   rN   cached_freqscached_scales)requires_graddummyr,   r7   g?gffffff?)super__init__rL   r   r   arangefloatlinspacer   onescache_if_possible	tmp_storer   	Parameterr4   learned_freqtensorseq_before_head_dimdefault_seq_diminterpolate_factoruse_xpos
scale_basestaticmethodr?   )selfr   rK   rL   thetamax_freq	num_freqsr]   rb   xpos_scale_basera   theta_rescale_factorr_   rZ   r4   r7   	__class__r   r   rU   Q   s>   
$zRotaryEmbedding.__init__c                 C   s   | j jS r   )rR   devicere   r   r   r   rm      s   zRotaryEmbedding.devicec                 C   s   | j ||dd d S )NF)
persistent)register_buffer)re   keyvaluer   r   r   r[         zRotaryEmbedding.tmp_storer   c                 C   s   t j|||d| | j S )N)rm   r.   )r   rV   ra   )re   r9   rm   r.   offsetr   r   r   get_seq_pos   s   zRotaryEmbedding.get_seq_posc                 C   sr   t || j}| jrJ d|j|j|j| }}}| j| j||||d||d}|dkr2t|d}t	|||dS )Nzyou must use `.rotate_queries_and_keys` method instead and pass in both queries and keys, for length extrapolatable rotary embeddings)rm   r.   rt   )r9   rt   rS   n d -> n 1 d)r8   )
r   r`   rb   rm   r.   r0   forwardru   r   r?   )re   r5   r8   rt   rm   r.   r9   r4   r   r   r   rotate_queries_or_keys   s   
z&RotaryEmbedding.rotate_queries_or_keysc           	      C   sv   t || j}|j| |j| }}||ksJ | j|||| | d}| j|||d}||j}||j}||fS )N)r8   rt   )r   r`   r0   rx   r3   r.   )	re   qkr8   rt   q_lenk_len	rotated_q	rotated_kr   r   r   rotate_queries_with_cached_keys   s   z/RotaryEmbedding.rotate_queries_with_cached_keysc                 C   s   t || j}| jsJ |j|j|j| }}}| j|||d}| j||d}| j||d	|}	|dkr?t
|d}t
|	d}	t|||	|d}
t|||	d |d}|
|j}
||j}|
|fS )N)r.   rm   r9   rS   rv   )r7   r8   r   )r   r`   rb   rm   r.   r0   ru   rw   	get_scaletor   r?   r3   )re   ry   rz   r8   rm   r.   r9   seqr4   r7   r}   r~   r   r   r   rotate_queries_and_keys   s   


z'RotaryEmbedding.rotate_queries_and_keysr5   r9   c                 C   s   | j sJ | jot|}|r&t| jr&|| | jjd kr&| j|||  S d}| j rG|t|d  | j }| jt|d }t	j
||fdd}|rO| d| |S )Nr   r+   r   zn -> n 1r   r   rP   )rb   rZ   r   rP   r0   lenrc   r7   r   r   r   r[   )re   r5   r9   rt   should_cacher7   powerr   r   r   r      s    
zRotaryEmbedding.get_scalec           
      G   s   t d }g }t|D ]=\}}| jdkrtjdd|| jd}ntj|| jd}| j||d}d gt| }|||< t	g||R }	|
||	  q
t| }tj|ddS )NrM   r   rI   )stepsrm   rm   r   r   )slice	enumeraterL   r   rX   rm   rV   rw   r   Ellipsisappendr
   r   )
re   dimsColon	all_freqsindr   posr4   all_axisnew_axis_slicer   r   r   get_axial_freqs   s   
zRotaryEmbedding.get_axial_freqsr(   r)   c                 C   s   | j o| j ot|o| jdk}|r,t| jr,|| | jjd kr,| j|||   S | j}td|	|j
|}t|ddd}|rJ| d|  |S )NrM   r   r@   rA   r   r    rO   )rZ   r]   r   rL   rO   r0   detachr4   r   r3   r.   r   r[   )re   r5   r9   rt   r   r4   r   r   r   rw     s&   zRotaryEmbedding.forward)NrF   rG   rH   rI   FFrJ   r+   r+   FT)r   )Nr   r   )__name__
__module____qualname__r   r	   r   r   rU   propertyrm   r[   ru   rx   r   r   intr   r   r   rw   __classcell__r   r   rk   r   rE   P   s>    M





rE   c                   @   s2   e Zd ZdZdddZdd Zdd Zd	d
 ZdS )Rope2DzA Helper class to apply RoPE2D as well as interpolate on the fly. Fc                 C   s   || _ || _d | _d | _d S r   )r   use_cls_token	grid_sizefreq)re   r   r   r   r   r   rU   2  s   
zRope2D.__init__c                 C   s   t | jd | _d S )Nr   )rE   r   ropern   r   r   r   init_tensors8  rs   zRope2D.init_tensorsc           	      C   s  | j ||fkr||f| _ | j|| _| jr)tj||dd }tj||dd }ntj||d}tj||d}| |d d d f ||d}| |d d d f ||d}tj||gdd|| d}| jr{tjtj	d|j
d |d|gdd}|d | _| j|| _d S )Nr   rI   r   r   r   )N.)r   r   r   r   r   rV   expandr   reshapezerosr0   r   )	re   rm   grid_hgrid_wgrid_y_rangegrid_x_rangefreqs_yfreqs_xr   r   r   r   update_grid;  s"   
  
zRope2D.update_gridc                 C   sP   t | jd d d d d d d f |}t | jd d d d d d d f |}||fS r   )r?   r   )re   ry   rz   r   r   r   __call__V  s   $$zRope2D.__call__N)F)r   r   r   __doc__rU   r   r   r   r   r   r   r   r   /  s    
r   )r   )r   r+   r,   )r   N)mathr   r   typingr   r   r   r   einopsr   r   r	   r
   r   r   	torch.ampr   torch.nnr   r   r   r   r   r'   r?   rD   rE   r   r   r   r   r   <module>   s"    


 `