o
    i!                     @   s   d dl Z d dlZd dlZd dlmZ G dd dejZdd Zedd ej	
ddd	 D Zed
kr>d dlZejd G dd dejZG dd dejZG dd dejZG dd deZdS )    Nc                       sR   e Zd Z fddZdd ZdejdejfddZd	d
 Zdd Z	dd Z
  ZS )MultiHeadedAttentionSANMExportc                    R   t    |j| _|j| _|j| _|j| _|j| _|j| _d | _| j| j | _	d S N)
super__init__d_kh
linear_outlinear_q_k_v
fsmn_blockpad_fnattnall_head_sizeselfmodel	__class__ T/home/ubuntu/.local/lib/python3.10/site-packages/funasr/models/sanm/multihead_att.pyr   	   s   
z'MultiHeadedAttentionSANMExport.__init__c                 C   s^   |\}}|  |\}}}}| ||}	|| jd  }t||dd}
| ||
|}||	 S )Ng      )forward_qkvforward_fsmnr   torchmatmul	transposeforward_attention)r   xmaskmask_3d_btdmask_4d_bhltq_hk_hv_hvfsmn_memoryscoresatt_outsr   r   r   forward   s   z&MultiHeadedAttentionSANMExport.forwardr   returnc                 C   6   |  d d | j| jf }||}|ddddS Nr   r            sizer   r   viewpermuter   r   new_x_shaper   r   r   transpose_for_scores      
z3MultiHeadedAttentionSANMExport.transpose_for_scoresc           	      C   sV   |  |}tj|t| j| j dd\}}}| |}| |}| |}||||fS Nr   dim)r
   r   splitintr   r   r6   )	r   r   q_k_vqkr%   r"   r#   r$   r   r   r   r   #   s   
"


z*MultiHeadedAttentionSANMExport.forward_qkvc                 C   sH   || }| dd}| |}| |}| dd}|| }|| }|S Nr.   r-   )r   r   r   )r   inputsr   r   r   r   r   r   +   s   

z+MultiHeadedAttentionSANMExport.forward_fsmnc                 C   b   || }t j|dd}t ||}|dddd }| d d | jf }||}| |S Nr   r9   r   r-   r.   r/   r   	r   softmaxr   r3   
contiguousr1   r   r2   r	   r   valuer'   r   r   context_layernew_context_layer_shaper   r   r   r   7      

z0MultiHeadedAttentionSANMExport.forward_attention)__name__
__module____qualname__r   r)   r   Tensorr6   r   r   r   __classcell__r   r   r   r   r      s    	r   c                 C   sf   | | } |  dd} |d u r|| } | |fS tj|| fdd} | d d d d |d  d f }| |fS )Nr.   r-   r9   )r   r   cat)r   r   cacher   kernel_sizer   r   r   preprocess_for_attnC   s    rT   c                 C   s   g | ]}t |qS r   )r<   ).0ir   r   r   
<listcomp>N   s    rW   .r-   )r.      c                       s&   e Zd Z fddZdddZ  ZS )%MultiHeadedAttentionSANMDecoderExportc                    s,   t    |j| _|j| _|j| _d | _d S r   )r   r   r   r   rS   r   r   r   r   r   r   V   s
   

z.MultiHeadedAttentionSANMDecoderExport.__init__Nc                 C   sF   t |||| j| j\}}| |}|dd}|| }|| }||fS r@   )rT   r   rS   r   r   )r   rA   r   rR   r   r   r   r   r)   ]   s   
z-MultiHeadedAttentionSANMDecoderExport.forwardr   )rL   rM   rN   r   r)   rP   r   r   r   r   rZ   U   s    rZ   c                       J   e Zd Z fddZdd ZdejdejfddZd	d
 Zdd Z	  Z
S )"MultiHeadedAttentionCrossAttExportc                    sJ   t    |j| _|j| _|j| _|j| _|j| _d | _| j| j | _d S r   )	r   r   r   r   linear_q
linear_k_vr	   r   r   r   r   r   r   r   h   s   
z+MultiHeadedAttentionCrossAttExport.__init__c                 C   s@   |  ||\}}}t||ddt| j }| |||S Nr   r   r   r   r   r   mathsqrtr   r   )r   r   memorymemory_maskr>   r?   r%   r'   r   r   r   r)   r   s    z*MultiHeadedAttentionCrossAttExport.forwardr   r*   c                 C   r+   r,   r0   r4   r   r   r   r6   w   r7   z7MultiHeadedAttentionCrossAttExport.transpose_for_scoresc                 C   s\   |  |}| |}tj|t| j| j dd\}}| |}| |}| |}|||fS r8   )r]   r^   r   r;   r<   r   r   r6   )r   r   rc   r>   k_vr?   r%   r   r   r   r   |   s   

 



z.MultiHeadedAttentionCrossAttExport.forward_qkvc                 C   rB   rC   rD   rG   r   r   r   r      rK   z4MultiHeadedAttentionCrossAttExport.forward_attentionrL   rM   rN   r   r)   r   rO   r6   r   r   rP   r   r   r   r   r\   g   s    

r\   c                       r[   )OnnxMultiHeadedAttentionc                    r   r   )
r   r   r   r   r]   linear_klinear_vr	   r   r   r   r   r   r   r      s   
z!OnnxMultiHeadedAttention.__init__c           	      C   sB   |  |||\}}}t||ddt| j }| |||S r_   r`   )	r   querykeyrH   r   r>   r?   r%   r'   r   r   r   r)      s    z OnnxMultiHeadedAttention.forwardr   r*   c                 C   r+   r,   r0   r4   r   r   r   r6      r7   z-OnnxMultiHeadedAttention.transpose_for_scoresc                 C   sF   |  |}| |}| |}| |}| |}| |}|||fS r   )r]   rh   ri   r6   )r   rj   rk   rH   r>   r?   r%   r   r   r   r      s   






z$OnnxMultiHeadedAttention.forward_qkvc                 C   rB   rC   rD   rG   r   r   r   r      rK   z*OnnxMultiHeadedAttention.forward_attentionrf   r   r   r   r   rg      s    	rg   c                       s4   e Zd Z fddZdd Zdd Zdd Z  ZS )	OnnxRelPosMultiHeadedAttentionc                    s(   t  | |j| _|j| _|j| _d S r   )r   r   
linear_pos
pos_bias_u
pos_bias_vr   r   r   r   r      s   z'OnnxRelPosMultiHeadedAttention.__init__c                 C   s   |  |||\}}}|dd}| | |}	|| j dd}
|| j dd}t|
|dd}t||	dd}| |}|| t	
| j }| |||S )Nr.   r-   r   r   )r   r   r6   rm   rn   ro   r   r   	rel_shiftra   rb   r   r   )r   rj   rk   rH   pos_embr   r>   r?   r%   pq_with_bias_uq_with_bias_v	matrix_ac	matrix_bdr'   r   r   r   r)      s   
z&OnnxRelPosMultiHeadedAttention.forwardc                 C   s   t jg | d d dR |j|jd}t j||gdd}|jg | d d |dd |dR  }|d d d d dd f |d d d d d d d |dd d f }|S )Nr/   r.   )devicedtyper   r9   r-   )r   zerosr1   rw   rx   rQ   r2   view_as)r   r   zero_padx_paddedr   r   r   rp      s   *4(z(OnnxRelPosMultiHeadedAttention.rel_shiftc                 C   rB   rC   rD   rG   r   r   r   r      rK   z0OnnxRelPosMultiHeadedAttention.forward_attention)rL   rM   rN   r   r)   rp   r   rP   r   r   r   r   rl      s
    
rl   )osra   r   torch.nnnnModuler   rT   tuple__version__r;   torch_versiontorch.fxfxwraprZ   r\   rg   rl   r   r   r   r   <module>   s    ;"++