o
    ߥi)                     @   s  d dl mZ d dlmZmZmZmZmZmZ d dl	Z	d dl
mZ d dlm  mZ d dlZ	d dlmZ d dlmZ G dd dejZG dd	 d	ejZG d
d dejZdd Ze	jjde	jde	jde	jde	jdede	jfddZe	jjde	jde	jde	jde	jfddZdd Ze	jjde	jde	jde	jde	jde	jde	jdede	jfddZ dd Z!G d d! d!ej"Z#d"ed#ee$ef d$e%d%e%def
d&d'Z&dS )(    )partial)AnyCallableDictIterableListOptionalN)	LayerNorm)tensor_tree_mapc                	       sX   e Zd Z		ddedededef fddZdd
dZdd ZdddZ	dd Z
  ZS )LinearTdefaultd_ind_outbiasinitc                    s   t t| j|||d || _| jr+t  | jd W d    n1 s&w   Y  |dkr6| d d S |dkrA| d d S |dkrK| 	  d S |dkrW| 
| j d S |d	kra|   d S |d
krl| 
d d S td)N)r   r   r         ?relug       @glorotgatingnormalfinalFzInvalid init method.)superr   __init__use_biastorchno_gradr   fill__trunc_normal_init_glorot_uniform_init
_zero_init_normal_init
ValueError)selfr   r   r   r   	__class__ d/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/models/science/unifold/modules/common.pyr      s&   
zLinear.__init__r   c                 C   sB   d}| j j\}}|td| }|d | }tjj| j d|d d S )Ng۶%?   g      ?        )meanstd)weightshapemaxnnr   trunc_normal_)r"   scaleTRUNCATED_NORMAL_STDDEV_FACTOR_fan_inr*   r%   r%   r&   r   /   s
   zLinear._trunc_normal_initc                 C   s   t jj| jdd d S )Nr'   )gain)r.   r   xavier_uniform_r+   r"   r%   r%   r&   r   7   s   zLinear._glorot_uniform_initc              	   C   s   t  ; | jd |r/t   | jd W d    n1 s"w   Y  W d    d S W d    d S W d    d S 1 sBw   Y  d S )Nr(   r   )r   r   r+   r   r   )r"   r   r%   r%   r&   r   :   s   

"zLinear._zero_initc                 C   s   t jjj| jdd d S )Nlinear)nonlinearity)r   r.   r   kaiming_normal_r+   r6   r%   r%   r&   r    A      zLinear._normal_init)Tr   )r   )T)__name__
__module____qualname__intboolstrr   r   r   r   r    __classcell__r%   r%   r#   r&   r      s     

r   c                       sh   e Zd Z fddZdd Zejjdejde	dejfdd	Z
	
ddejdee	 dejfddZ  ZS )
Transitionc                    sf   t t|   || _|| _t| j| _t| j| j| j dd| _t	
 | _t| j| j |dd| _d S )Nr   r   r   )r   rB   r   r   nr	   
layer_normr   linear_1r.   GELUactlinear_2)r"   r   rD   r#   r%   r&   r   G   s   
zTransition.__init__c                 C   s,   |  |}| |}| |}| |}|S N)rE   rF   rH   rI   )r"   xr%   r%   r&   _transitionS   s
   



zTransition._transitionrK   
chunk_sizereturnc                 C   s$   t | jd|i|t|jd d dS )NrK   rM   num_batch_dims)chunk_layerrL   lenr,   r"   rK   rM   r%   r%   r&   _chunkZ   s   zTransition._chunkNc                 C   s(   |d ur|  ||}|S | j|d}|S )N)rK   )rU   rL   rT   r%   r%   r&   forwardg   s
   zTransition.forwardrJ   )r;   r<   r=   r   rL   r   jitignoreTensorr>   rU   r   rV   rA   r%   r%   r#   r&   rB   E   s&    rB   c                	       s   e Zd Zd fdd	Zdd Zejjdejdejde	d	ejfd
dZ
dd Z		ddejdeej dee	 d	ejfddZ  ZS )OuterProductMeanMbP?c                    s   t t|   || _|| _|| _|| _t|| _t	||| _
t	||| _t	|d |dd| _t | _t	| j| jdd| _t| j| _d S )N   r   rC   r   )r   rZ   r   d_msad_paird_hidepsr	   rE   r   rF   rI   
linear_outr.   rG   rH   linear_zlayer_norm_out)r"   r]   r^   r_   r`   r#   r%   r&   r   w   s   

zOuterProductMean.__init__c                 C   s4   t d||}||jd d d }| |}|S )Nz...bac,...dae->...bdcerO   )r   einsumreshaper,   ra   )r"   abouterr%   r%   r&   _opm   s   
zOuterProductMean._opmrh   ri   rM   rN   c                 C   s   | d|jdd   }| d|jdd   }g }t||D ]\}}tt| j|dd|i|dd}|| qt|dkrF|d d}nt	j
|dd}| |jd d |jdd   }|S )	Nrd   )ri   rh   r'   rP   r   )dim)rg   r,   ziprR   r   rk   appendrS   	unsqueezer   stack)r"   rh   ri   rM   outa_primeb_primerj   r%   r%   r&   rU      s    "zOuterProductMean._chunkc                 C   s   d | _ d | _d S rJ   )rb   rc   r6   r%   r%   r&   apply_alphafold_original_mode   s   
z.OuterProductMean.apply_alphafold_original_modeNmmaskc                 C   s   |  |}|d}| jd ur||dd  }| |}| |}| jr.|| }|| }n||9 }||9 }|dd}|dd}|d urN| |||}n| 	||}t
d||}|| j|  }| jd urv| |}| |}| |}|S )Nre   rO   g      rl   z...abc,...adc->...bdc)rE   rp   rc   sizerF   rI   training	transposerU   rk   r   rf   r`   rH   rb   )r"   rv   rw   rM   rh   ri   znormr%   r%   r&   rV      s.   









zOuterProductMean.forward)r[   )NN)r;   r<   r=   r   rk   r   rW   rX   rY   r>   rU   ru   r   rV   rA   r%   r%   r#   r&   rZ   u   s*    rZ   c                 C   s   |r||  S | |7 } | S rJ   r%   )residualrK   ry   r%   r%   r&   r}      s   r}   rK   r   dropmaskprobrN   c                 C   s   | | t j||dd | S NT)pry   )Fdropout)rK   r   r}   r~   r   r%   r%   r&   fused_bias_dropout_add   s   r   c                 C   s   |||  7 }|S rJ   r%   )rK   r   r}   r%   r%   r&    fused_bias_dropout_add_inference   s   r   c           	      C   sl   |   }|r0t|j}d||< t  ||}W d    n1 s#w   Y  t|||||S t|||S Nr'   )get_output_biaslistr,   r   r   new_onesr   r   )	moduler}   rK   dropout_shared_dimr   ry   r   r,   rw   r%   r%   r&   bias_dropout_residual   s   

r   gg_biasdropout_maskc                 C   s*   t || | |  tj||dd | S r   )r   sigmoidr   r   )rK   r   r   r   r}   r   r   r%   r%   r&   fused_bias_gated_dropout_add   s   
r   c                 C   s   |r8|\}}|   \}	}
t|j}d||< t  ||}W d    n1 s)w   Y  t||	||
|||S |d u rU|\}}|   \}	}
|t||
 ||	  7 }|S ||7 }|S r   )r   r   r,   r   r   r   r   r   )r   r}   outputsr   r   ry   
block_sizerK   r   r   r   r,   rw   r%   r%   r&   tri_mul_residual	  s0   	

	r   c                   @   s   e Zd Zdd ZdS )SimpleModuleListc                 C   s   t t| d | d   S )Nz X ...
r   )r@   rS   __repr__r6   r%   r%   r&   r   /  s   zSimpleModuleList.__repr__N)r;   r<   r=   r   r%   r%   r%   r&   r   -  s    r   layerinputsrM   rQ   c                    sr  t |dks
td fdd dd | D }fdd |D }td	d t| D d
D ]}|9 q3| d
 | }fdd}t||}d }	t|D ]W}
|
| t|
d
 | fdd}t||}| di |}|	d u rtfdd|}	t|}|tu rt|	|D ]
\}}||< qqT|t	j
u r||	< qTtdfdd}t||	}	|	S )Nr   zMust provide at least one inputc                    s|   g }t | tju r|| j |S t | tu r'|  D ]	}| | q|S t| t	r:| D ]	}| | q.|S t
d)NNot supported)typer   rY   ro   r,   dictvaluesextend
isinstancer   r!   )inputshapesv)_dict_get_shapesr%   r&   r   =  s   

z%chunk_layer.<locals>._dict_get_shapesc                 S   s   i | ]\}}|d ur||qS rJ   r%   ).0kr   r%   r%   r&   
<dictcomp>L  s    zchunk_layer.<locals>.<dictcomp>c                    s   g | ]}|d   qS rJ   r%   )r   r,   )rQ   r%   r&   
<listcomp>M  s    zchunk_layer.<locals>.<listcomp>c                 S   s   g | ]}t |qS r%   )r-   )r   sr%   r%   r&   r   P  s    r'   c                    sD   | j dg| jd  R  } | jd  ks | jd dks J d| S )Nre   r   r'   z>batch dimension must be 1 or equal to the flat batch dimensionviewr,   t)flat_batch_dimrQ   r%   r&   _flat_inputsW  s
   z!chunk_layer.<locals>._flat_inputsc                    s&   | j d dkr| dd S |   S )Nr   r'   )r,   r   )	chunk_endchunk_startr%   r&   select_chunke  s   z!chunk_layer.<locals>.select_chunkc                    s   |   f| jdd   S r   )	new_zerosr,   r   )r   r%   r&   <lambda>q  s    zchunk_layer.<locals>.<lambda>r   c                    s   |   | jdd   S r   r   r   )orig_batch_dimsr%   r&   rg   ~  r:   zchunk_layer.<locals>.reshaper%   )rS   r!   itemstuplern   r
   rangeminr   r   rY   )r   r   rM   rQ   initial_dimsd
num_chunksr   flat_inputsrr   ir   chunkesoutput_chunkout_typerK   yrg   r%   )r   r   r   r   rQ   r   r&   rR   3  sJ   






rR   )'	functoolsr   typingr   r   r   r   r   r   r   torch.nnr.   torch.nn.functional
functionalr   torch.utils.checkpointunicore.modulesr	   unicore.utilsr
   r   ModulerB   rZ   r}   rW   scriptrY   floatr   r   r   r   r   
ModuleListr   r@   r>   rR   r%   r%   r%   r&   <module>   s    60Y
	$
