o
    ´©i3  ã                   @   s|   d dl Z d dlZd dlm  mZ d dlmZ G dd„ dejƒZG dd„ dejƒZG dd„ dejjƒZ	G d	d
„ d
ejƒZ
dS )é    N)Únnc                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )ÚMultiHeadSelfAttentioné   çš™™™™™¹?c                    sb   t ƒ  ¡  t ||¡| _t ||¡| _t ||¡| _t ||¡| _|| | _|| _	t 
|¡| _d S ©N)ÚsuperÚ__init__r   ÚLinearÚlinearQÚlinearKÚlinearVÚlinearOÚd_kÚhÚDropoutÚdropout)ÚselfÚn_unitsr   Údropout_rate©Ú	__class__© úN/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/eend/encoder.pyr   	   s   

zMultiHeadSelfAttention.__init__c           	   	   C   s  |   |¡ |d| j| j¡}|  |¡ |d| j| j¡}|  |¡ |d| j| j¡}t | dddd¡| dddd¡¡t	 
| j¡ }|d urP| d¡}| |dkd¡}tj|dd| _|  | j¡}t || dddd¡¡}| dddd¡ ¡  d| j| j ¡}|  |¡S )Néÿÿÿÿr   é   é   é   g    eÍÍÁ)Údim)r
   Úviewr   r   r   r   ÚtorchÚmatmulÚpermuteÚmathÚsqrtÚ	unsqueezeÚmasked_fillÚFÚsoftmaxÚattr   Ú
contiguousr   )	r   ÚxÚ
batch_sizeÚx_maskÚqÚkÚvÚscoresÚp_attr   r   r   Ú__call__   s   0
$
zMultiHeadSelfAttention.__call__)r   r   ©Ú__name__Ú
__module__Ú__qualname__r   r2   Ú__classcell__r   r   r   r   r      s    
r   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚPositionwiseFeedForwardc                    s:   t t| ƒ ¡  t ||¡| _t ||¡| _t |¡| _d S r   )	r   r8   r   r   r	   Úlinear1Úlinear2r   r   )r   r   Úd_unitsr   r   r   r   r   #   s   z PositionwiseFeedForward.__init__c              	   C   s   |   |  t |  |¡¡¡¡S r   )r:   r   r&   Úrelur9   ©r   r*   r   r   r   r2   )   s   z PositionwiseFeedForward.__call__r3   r   r   r   r   r8   "   s    r8   c                       s6   e Zd Zd
‡ fdd„	Zdd„ Zdejfdd	„Z‡  ZS )ÚPositionalEncodingéˆ  Fc                    sZ   t t| ƒ ¡  || _|| _t | j¡| _tj	j
|d| _d | _|  t d¡ d|¡¡ d S )N)Úpg        r   )r   r>   r   Úd_modelÚreverser"   r#   Úxscaler   r   r   r   ÚpeÚ	extend_peÚtensorÚexpand)r   rA   r   Úmax_lenrB   r   r   r   r   .   s   zPositionalEncoding.__init__c                 C   sD  | j d ur+| j  d¡| d¡kr+| j j|jks| j j|jkr)| j j|j|jd| _ d S t | d¡| j¡}| jrKtj	| d¡d ddtj
d d¡}ntj	d| d¡tj
d d¡}t tj	d| jdtj
dt d¡| j   ¡}t || ¡|d d …dd d…f< t || ¡|d d …dd d…f< | d¡}|j|j|jd	| _ d S )
Nr   )ÚdtypeÚdevicer   g      ð¿)rI   r   r   g     ˆÃ@)rJ   rI   )rD   ÚsizerI   rJ   Útor   ÚzerosrA   rB   ÚarangeÚfloat32r$   Úexpr"   ÚlogÚsinÚcos)r   r*   rD   ÚpositionÚdiv_termr   r   r   rE   7   s$   
&ÿÿ  
zPositionalEncoding.extend_per*   c                 C   s:   |   |¡ || j | jd d …d | d¡…f  }|  |¡S )Nr   )rE   rC   rD   rK   r   r=   r   r   r   ÚforwardK   s   
&
zPositionalEncoding.forward)r?   F)	r4   r5   r6   r   rE   r   ÚTensorrV   r7   r   r   r   r   r>   -   s    	r>   c                       sN   e Zd Z				ddedededed	ed
edef‡ fdd„Zddd„Z‡  ZS )ÚEENDOLATransformerEncoderé   é   r   FÚidimÚn_layersr   Úe_unitsr   r   Úuse_pos_embc           	         sÂ   t t| ƒ ¡  t ||¡| _t |¡| _|| _t 	|¡| _
t|ƒD ]7}t| d d|¡t |¡ƒ t| d d|¡t||ƒƒ t| d d|¡t |¡ƒ t| d d|¡t|||ƒƒ q!t |¡| _d S )Nú{}{:d}Úlnorm1_Ú	self_att_Úlnorm2_Úff_)r   rX   r   r   r	   Ú	linear_inÚ	LayerNormÚlnorm_inr\   r   r   ÚrangeÚsetattrÚformatr   r8   Ú	lnorm_out)	r   r[   r\   r   r]   r   r   r^   Úir   r   r   r   R   s   


ýz"EENDOLATransformerEncoder.__init__Nc                 C   s¾   |j d |j d  }|  | |d¡¡}t| jƒD ]A}t| d d|¡ƒ|ƒ}t| d d|¡ƒ||j d |ƒ}||  |¡ }t| d d|¡ƒ|ƒ}t| d d|¡ƒ|ƒ}||  |¡ }q|  |¡S )	Nr   r   r   r_   r`   ra   rb   rc   )	Úshaperd   Úreshaperg   r\   Úgetattrri   r   rj   )r   r*   r,   ÚBT_sizeÚerk   Úsr   r   r   r2   l   s    
z"EENDOLATransformerEncoder.__call__)rY   rZ   r   Fr   )	r4   r5   r6   ÚintÚfloatÚboolr   r2   r7   r   r   r   r   rX   Q   s*    øþýüûúùørX   )r"   r   Útorch.nn.functionalr   Ú
functionalr&   ÚModuler   r8   r>   rX   r   r   r   r   Ú<module>   s    $