o
    ih                     @   s   d dl Z d dlmZ d dlm  mZ d dl mZ d dlmZ dd Z	e	e j
ZdddZeejddd	Zeejddd	ZG d
d dejZG dd dejZe jjdd ZG dd dejZe jjdd ZG dd dejZdS )    N)Tensor)weight_normc                 C   s   t | jS N)torchfinfoeps)	data_type r	   K/home/ubuntu/.local/lib/python3.10/site-packages/neucodec/distill_layers.pyget_eps      r   Tc                    s    fdd}|S )Nc                     sD   | i |} rt jj|jdd t j|jd r t|}|S )Ng{Gz?)stdr   )nninittrunc_normal_weight	constant_biasr   )argskwargsnn_instanceinit_weightnn_classnorm_weightr	   r
   
nn_builder   s   znn_wrapper.<locals>.nn_builderr	   )r   r   r   r   r	   r   r
   
nn_wrapper   s   	r   )r   r   c                       sL   e Zd Z	ddejdedef fddZdefd	d
Z	defddZ
  ZS )Residual        Tmodule	drop_probscale_by_keepc                    s<   t    d|  krdk sJ  J || _|| _|| _d S )Nr      )super__init__r   r    r!   )selfr   r    r!   	__class__r	   r
   r$   !   s
   

zResidual.__init__x_sidec                 C   s`   | j dks| js
|S d| j  }|jd fd|jd   }|||}| jr,|| || S )Nr   r"   r   r"   )r    trainingshapendim	new_empty
bernoulli_r!   div_)r%   r(   	keep_probr+   	keep_maskr	   r	   r
   	drop_path*   s   

zResidual.drop_pathxc                 C   s   |  |}| |}|| S r   )r   r2   )r%   r3   r(   r	   r	   r
   forward4   s   

zResidual.forward)r   T)__name__
__module____qualname__r   Modulefloatboolr$   r   r2   r4   __classcell__r	   r	   r&   r
   r       s    	
r   c                       6   e Zd ZdZedf fdd	Zdd Zdd Z  ZS )	GRNa.  GRN (Global Response Normalization) layer
    Which supports two data formats: channels_last (default) or channels_first.
    Channels_last corresponds to inputs with shape (batch_size, Sequence, channels)
    while channels_first corresponds to inputs with shape (batch_size, channels, Sequence).
    channels_lastc                    s   t    || _|| _|dkr'ttd|| _ttd|| _	d| _
n#|dkrCtt|d| _tt|d| _	d| _
ntd| t|| _d S )Nr>   r"   channels_firstzUnsupported data_format: )r#   r$   
n_channelsdata_formatr   	Parameterr   zerosgammabetachannel_dim
ValueErrortensorr   r%   rA   r   rB   r&   r	   r
   r$   A   s   
zGRN.__init__c                 C   sH   t j|dddgdd}||j| jdd| j  }| j||  | j | S )N   r"   T)pdimkeepdim)rM   rN   )r   normmeanrG   r   rE   rF   )r%   r3   g_xn_xr	   r	   r
   r4   Q   s   zGRN.forwardc                 C      | j j d| j d| j dS Nz(n_channels=z, )r'   r5   rA   rB   r%   r	   r	   r
   __repr__V      zGRN.__repr__	r5   r6   r7   __doc__EPSr$   r4   rX   r;   r	   r	   r&   r
   r=   :   s
    r=   c                 C   s,   d}| ||   t||  d  } | S )Ng      >rK   )
reciprocalr   sinpow)r3   alphar   r	   r	   r
   snake[   s   $ra   c                       s&   e Zd Zd fdd	Zdd Z  ZS )Snake1dr@   c                    sR   t    |dkrttd|d| _d S |dkr'ttdd|| _d S t)Nr@   r"   r>   )r#   r$   r   rC   r   onesr`   NotImplementedError)r%   channelsrB   r&   r	   r
   r$   d   s   
zSnake1d.__init__c                 C   s   t || jS r   )ra   r`   )r%   r3   r	   r	   r
   r4   m   r   zSnake1d.forward)r@   )r5   r6   r7   r$   r4   r;   r	   r	   r&   r
   rb   c   s    	rb   c                 C   sL   | j ddd}| | dj ddd}| | t||  } ||  | } | S )Nr"   T)rN   rK   )rP   r_   r   sqrt)r3   r   r   r   usr	   r	   r
   channel_normq   s
   ri   c                       r<   )	ChannelNorma  ChannelNorm that supports two data formats: channels_last (default) or channels_first.
    Channels_last corresponds to inputs with shape (batch_size, ..., channels)
    while channels_first corresponds to inputs with shape (batch_size, channels, ...).
    r>   c                    sJ   t    || _|| _tt|| _tt	|| _
t|| _d S r   )r#   r$   rA   rB   r   rC   r   rc   r   rD   r   rI   r   rJ   r&   r	   r
   r$      s   
zChannelNorm.__init__c                 C   s~   | j dkr(dt|jdd   }t|| jjdg|R  | jjdg|R  | jS | j dkr=t	|| j
f| j| j| j S t)Nr@   r)   rK   r?   r>   )rB   lenr+   ri   r   viewr   r   F
layer_normrA   itemrd   )r%   r3   extend_dimsr	   r	   r
   r4      s   

zChannelNorm.forwardc                 C   rS   rT   rV   rW   r	   r	   r
   rX      rY   zChannelNorm.__repr__rZ   r	   r	   r&   r
   rj   z   s
    rj   )TT)r   torch.nnr   torch.nn.functional
functionalrm   r   torch.nn.utils.parametrizationsr   r   float32r\   r   Conv1dLinearr8   r   r=   jitscriptra   rb   ri   rj   r	   r	   r	   r
   <module>   s$    

!

