o
    ߥi#0                     @   s  d dl mZ d dlmZmZ d dlZd dlmZ d dlm	Z	m
Z
 d dlmZ ddlmZmZ dd	 ZG d
d dejZG dd dejZd!ddZG dd dejZG dd deZG dd deZG dd dejZdd ZG dd dejZG dd deZG dd  d eZdS )"    )partialmethod)ListOptionalN)	LayerNormsoftmax_dropout)permute_final_dims   )Linearchunk_layerc                 C   s&   |dk sJ t | }||| dk< |S )Ng     r   )torch
zeros_like)maskneg_inf	attn_mask r   h/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/science/unifold/modules/attentions.pygen_attn_mask   s   
r   c                       sz   e Zd Z	ddedededededef fdd	Z	
	
ddejdejdejdejdeej dejfddZ	dd Z
  ZS )	AttentionTq_dimk_dimv_dimhead_dim	num_headsgatingc                    s   t t|   || _|| j }|| _t||ddd| _t||ddd| _t||ddd| _t||dd| _	d | _
| jrCt||dd| _
|d | _d S )NFglorotbiasinitfinalr   r         )superr   __init__r   r   r	   linear_qlinear_klinear_vlinear_olinear_gnorm)selfr   r   r   r   r   r   	total_dim	__class__r   r   r"      s   	
zAttention.__init__Nqkvr   r   returnc           	      C   sP  d }| j d ur|  |}| |}|| j9 }| |}| |}||jd d | jdf dd	 }||jd d | jdf dd	 }||jd d | jdf dd}t
||dd}~~t|d| j||d}t
||}~~|dd	 }|jg |jd d dR  }|d urt
|| }tj|| jj}|S )Nr   )r   r   )r'   r#   r(   r$   r%   viewshaper   	transpose
contiguousr   matmulr   trainingsigmoidnn
functionallinearr&   weight)	r)   r-   r.   r/   r   r   gattnor   r   r   forward0   s:   





&zAttention.forwardc                 C   s   | j jS N)r&   r   r)   r   r   r   get_output_biasY   s   zAttention.get_output_biasTNN)__name__
__module____qualname__intboolr"   r   Tensorr   rB   rE   __classcell__r   r   r+   r   r      s>    	
)r   c                       s8   e Zd Z fddZdejdejdejfddZ  ZS )GlobalAttentionc                    s   t t|   || _|| _|| _t||| ddd| _t||ddd| _t||ddd| _	t||| dd| _
t|| |dd| _t | _|d | _d S )NFr   r   r   r   r   r    )r!   rO   r"   r   infepsr	   r#   r$   r%   r'   r&   r;   Sigmoidr:   r(   )r)   	input_dimr   r   rP   rQ   r+   r   r   r"   _   s   
zGlobalAttention.__init__xr   r0   c           
      C   s0  |  | |}| |}| |}tj||d ddtj|ddd| j  }| |}|| j	9 }|
|jd d | jdf }t||dd}~~t|| j dd d d d d f }t|d| j|d}t||}	~~|
|jd d | jdf }|	d	| }	~|	|	jd d d
 }	| |	S )Nr1   r2   dimT)rV   keepdims.r   r   r3   )r1   )r:   r'   r$   r%   r   sum	unsqueezerQ   r#   r(   r4   r5   r   r8   r6   r   rP   r   r9   reshaper&   )
r)   rT   r   r?   r.   r/   r-   r@   r   rA   r   r   r   rB   o   s2   



"
zGlobalAttention.forward)rH   rI   rJ   r"   r   rM   rB   rN   r   r   r+   r   rO   ]   s    $rO   Tc                 C   s\   t | | dd d d d d d f }|r,t | dd| dd d d d d d f }||fS |S N.r1   r2   r   r6   )r   rP   gen_col_maskrow_maskcol_maskr   r   r   gen_msa_attn_mask   s   "ra   c                       s   e Zd Z		d fdd	Zejj			ddejdeej deej de	d	ejf
d
dZ
ejj			ddejdeej deej dee	 d	ejf
ddZddeej fddZ			ddejdeej deej dee	 d	ejf
ddZdd Z  ZS )MSAAttentionFNc                    sb   t t|   || _t|| _d | _d | _| jr&t|| _t||ddd| _t	|||||| _
d S NFnormalr   )r!   rb   r"   	pair_biasr   layer_norm_mlayer_norm_zlinear_zr	   r   mha)r)   d_ind_hidr   re   d_pairr+   r   r   r"      s   

zMSAAttention.__init__mr   r   
chunk_sizer0   c                 C   s(   t | j|||d|t|jd d dS )N)rm   r   r   r2   rn   num_batch_dimsr
   _attn_forwardlenr5   )r)   rm   r   r   rn   r   r   r   _chunk   s   	zMSAAttention._chunk 
  c              
   C   s   |  |}|jd | d | }g }t|D ]B}|| }t|jd || }	|d||	d d d d f }
|d urI|d||	d d d d d d f nd }|| j|
|
|
||d qtj|ddS )Nr3   r   .r-   r.   r/   r   r   rU   )rf   r5   rangeminappendri   r   cat)r)   rm   r   r   rn   	num_chunkoutputsichunk_start	chunk_endcur_mcur_maskr   r   r   _attn_chunk_forward   s   
"z MSAAttention._attn_chunk_forwardc                 C   s   |  |}| j|||||dS )Nrv   )rf   ri   )r)   rm   r   r   r   r   r   rr      s   
zMSAAttention._attn_forwardzr   c                 C   s   d }| j r| |}t| |dd }|d ur%| ||||}|S d}|jd |kr7| |||}|S | j	||||dS )N   r   r   ru   r3   )rn   )
re   rg   r   rh   rZ   r7   rt   r5   rr   r   )r)   rm   r   r   rn   r   attn_chunk_sizer   r   r   rB      s$   



zMSAAttention.forwardc                 C   
   | j  S rC   ri   rE   rD   r   r   r   rE         
zMSAAttention.get_output_bias)FNNNN)NNru   rC   )rH   rI   rJ   r"   r   jitignorerM   r   rK   rt   r   rr   rB   rE   rN   r   r   r+   r   rb      sd    
rb   c                       s   e Zd Z fddZ  ZS )MSARowAttentionWithPairBiasc                    s   t t| j|||d|d d S )NT)re   rl   )r!   r   r"   )r)   d_msarl   rk   r   r+   r   r   r"        

z$MSARowAttentionWithPairBias.__init__)rH   rI   rJ   r"   rN   r   r   r+   r   r     s    r   c                	       sN   e Zd Z fddZ		d
dejdeej dee dejf fdd	Z  Z	S )MSAColumnAttentionc                    s   t t| j|||dd d d S )NF)rj   rk   r   re   rl   )r!   r   r"   )r)   r   rk   r   r+   r   r   r"     r   zMSAColumnAttention.__init__Nrm   r   rn   r0   c                    s.   | dd}t j|||d}| dd}|S )Nr2   r3   )r   rn   )r6   r!   rB   )r)   rm   r   rn   r+   r   r   rB     s   zMSAColumnAttention.forwardrG   )
rH   rI   rJ   r"   r   rM   r   rK   rB   rN   r   r   r+   r   r     s    r   c                	       s   e Zd Z		d fdd	Zejjdejdejdedejfd	d
Z	dd Z
		ddejdeej dee dejfddZ  ZS )MSAColumnGlobalAttention    eA绽|=c                    s0   t t|   t|| _t|||||d| _d S )N)rP   rQ   )r!   r   r"   r   rf   rO   global_attention)r)   rj   rk   r   rP   rQ   r+   r   r   r"   (  s   
z!MSAColumnGlobalAttention.__init__rm   r   rn   r0   c                 C   s&   t | j||d|t|jd d dS )N)rm   r   r2   ro   rq   r)   rm   r   rn   r   r   r   rt   ;  s   zMSAColumnGlobalAttention._chunkc                 C   s   |  |}| j||dS )NrX   )rf   r   )r)   rm   r   r   r   r   rr   L  s   
z&MSAColumnGlobalAttention._attn_forwardNc                 C   sN   | dd}| dd}|d ur| |||}n| j||d}| dd}|S )Nr2   r3   r1   rX   )r6   rt   rr   r   r   r   r   rB   P  s   z MSAColumnGlobalAttention.forward)r   r   rG   )rH   rI   rJ   r"   r   r   r   rM   rK   rt   rr   r   rB   rN   r   r   r+   r   r   &  s4    r   c                 C   sT   t | | dd d d d d d f }t | dd| dd d d d d d f }||fS r\   r]   )r   rP   
start_maskend_maskr   r   r   gen_tri_attn_maskc  s
   "r   c                       s   e Zd Z fddZejj			ddejdeej deej de	dejf
d	d
Z
		ddejdeej dee	 dejfddZdd Z  ZS )TriangleAttentionc                    sF   t t|   || _t|| _t||ddd| _t|||||| _	d S rc   )
r!   r   r"   startingr   
layer_normr	   r=   r   ri   )r)   rj   rk   r   r   r+   r   r   r"   l  s
   
zTriangleAttention.__init__NrT   r   r   rn   r0   c                 C   s,   t | j|||||d|t|jd d dS )Nrv   r2   ro   )r
   ri   rs   r5   )r)   rT   r   r   rn   r   r   r   rt   y  s   zTriangleAttention._chunkr   c                 C   sz   | j s	|dd}| |}t| |dd }|d ur(| ||||}n
| j|||||d}| j s;|dd}|S )Nr2   r3   r   r   rv   )	r   r6   r   r   r=   rZ   r7   rt   ri   )r)   rT   r   rn   triangle_biasr   r   r   rB     s   


zTriangleAttention.forwardc                 C   r   rC   r   rD   r   r   r   rE     r   z!TriangleAttention.get_output_biasr   rG   )rH   rI   rJ   r"   r   r   r   rM   r   rK   rt   rB   rE   rN   r   r   r+   r   r   j  s:    
r   c                   @      e Zd ZeejddZdS )TriangleAttentionStartingTr   NrH   rI   rJ   r   r   r"   r   r   r   r   r         r   c                   @   r   )TriangleAttentionEndingFr   Nr   r   r   r   r   r     r   r   rF   )	functoolsr   typingr   r   r   torch.nnr;   unicore.modulesr   r   unicore.utilsr   commonr	   r
   r   Moduler   rO   ra   rb   r   r   r   r   r   r   r   r   r   r   r   <module>   s&   G
6
e=?