o
    پi                     @   s  d Z ddlZddlmZ ddlmZ d3defddZG d	d
 d
ejZ	d3defddZ
G dd dejZd3defddZG dd dejZd3defddZG dd dejZd3defddZG dd dejZd3defddZG dd dejZd3defdd ZG d!d" d"ejZG d#d$ d$ejZd3d%ejded&ejfd'd(ZG d)d* d*ejZd3d%ejded&ejfd+d,ZG d-d. d.ejZd3d%ejded&ejfd/d0ZG d1d2 d2ejZdS )4z Activations

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalFinplacec                 C   s    |r	|  |  S | |  S )z;Swish - Described in: https://arxiv.org/abs/1710.05941
    )mul_sigmoidmulxr    r
   K/home/ubuntu/.local/lib/python3.10/site-packages/timm/layers/activations.pyswish   s    r   c                       ,   e Zd Zddef fddZdd Z  ZS )SwishFr   c                       t t|   || _d S N)superr   __init__r   selfr   	__class__r
   r   r         
zSwish.__init__c                 C      t || jS r   )r   r   r   r	   r
   r
   r   forward      zSwish.forwardF__name__
__module____qualname__boolr   r   __classcell__r
   r
   r   r   r          r   c                 C   s   |  t|  S )zMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    NOTE: I don't have a working inplace variant
    )r   Fsoftplustanhr   r
   r
   r   mish   s   r'   c                       s0   e Zd ZdZddef fddZdd Z  ZS )	MishziMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    Fr   c                       t t|   d S r   )r   r(   r   r   r   r
   r   r   '      zMish.__init__c                 C      t |S r   )r'   r   r
   r
   r   r   *      zMish.forwardr   )r   r   r    __doc__r!   r   r   r"   r
   r
   r   r   r(   $   s    r(   c                 C      |r|   S |  S r   )sigmoid_r   r   r
   r
   r   r   .      r   c                       r   )SigmoidFr   c                    r   r   )r   r1   r   r   r   r   r
   r   r   4   r   zSigmoid.__init__c                 C      | j r| S | S r   )r   r/   r   r   r
   r
   r   r   8      zSigmoid.forwardr   r   r
   r
   r   r   r1   3   r#   r1   c                 C   r.   r   )tanh_r&   r   r
   r
   r   r&   <   r0   r&   c                       r   )TanhFr   c                    r   r   )r   r5   r   r   r   r   r
   r   r   B   r   zTanh.__init__c                 C   r2   r   )r   r4   r&   r   r
   r
   r   r   F   r3   zTanh.forwardr   r   r
   r
   r   r   r5   A   r#   r5   c                 C   s,   t | d d}|r| |S | |S )N      @      @)r$   relu6div_r   r   )r	   r   innerr
   r
   r   
hard_swishJ   s   r;   c                       r   )	HardSwishFr   c                    r   r   )r   r<   r   r   r   r   r
   r   r   P   r   zHardSwish.__init__c                 C   r   r   )r;   r   r   r
   r
   r   r   T   r   zHardSwish.forwardr   r   r
   r
   r   r   r<   O   r#   r<   c                 C   s.   |r|  ddddS t| d d S )Nr6   g        r7   )add_clamp_r9   r$   r8   r   r
   r
   r   hard_sigmoidX   s   r?   c                       r   )HardSigmoidFr   c                    r   r   )r   r@   r   r   r   r   r
   r   r   `   r   zHardSigmoid.__init__c                 C   r   r   )r?   r   r   r
   r
   r   r   d   r   zHardSigmoid.forwardr   r   r
   r
   r   r   r@   _   r#   r@   c                 C   s:   |r|  d| d jddd S d|  | d jddd S )z Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    g      ?   r   )minmax)r   clampr   r
   r
   r   	hard_mishh   s   rE   c                       r   )HardMishFr   c                    r   r   )r   rF   r   r   r   r   r
   r   r   t   r   zHardMish.__init__c                 C   r   r   )rE   r   r   r
   r
   r   r   x   r   zHardMish.forwardr   r   r
   r
   r   r   rF   s   r#   rF   c                	       sJ   e Zd ZdZddedededd	f fd
dZdej	dej	fddZ
  ZS )PReLUz)Applies PReLU (w/ dummy inplace arg)
             ?Fnum_parametersinitr   returnNc                    s   t t| j||d d S )N)rJ   rK   )r   rG   r   )r   rJ   rK   r   r   r
   r   r      s   zPReLU.__init__inputc                 C   s   t || jS r   )r$   preluweightr   rM   r
   r
   r   r         zPReLU.forward)rH   rI   F)r   r   r    r-   intfloatr!   r   torchTensorr   r"   r
   r
   r   r   rG   |   s     rG   r	   rL   c                 C   s
   t | S r   r$   gelur   r
   r
   r   rW         
rW   c                       >   e Zd ZdZd
def fddZdejdejfdd	Z  Z	S )GELULApplies the Gaussian Error Linear Units function (w/ dummy inplace arg)
    Fr   c                    r)   r   )r   rZ   r   r   r   r
   r   r      r*   zGELU.__init__rM   rL   c                 C   s
   t |S r   rV   rP   r
   r
   r   r      rX   zGELU.forwardr   
r   r   r    r-   r!   r   rT   rU   r   r"   r
   r
   r   r   rZ          rZ   c                 C   s   t j| ddS Nr&   )approximaterV   r   r
   r
   r   	gelu_tanh   rQ   r`   c                       rY   )GELUTanhr[   Fr   c                    r)   r   )r   ra   r   r   r   r
   r   r      r*   zGELUTanh.__init__rM   rL   c                 C   s   t j|ddS r^   rV   rP   r
   r
   r   r      rQ   zGELUTanh.forwardr   r\   r
   r
   r   r   ra      r]   ra   c                 C   s   | t d|   S )NgZd;?)rT   r   r   r
   r
   r   
quick_gelu   r*   rb   c                       rY   )	QuickGELUr[   Fr   c                    r)   r   )r   rc   r   r   r   r
   r   r      r*   zQuickGELU.__init__rM   rL   c                 C   r+   r   )rb   rP   r
   r
   r   r      r,   zQuickGELU.forwardr   r\   r
   r
   r   r   rc      r]   rc   r   )r-   rT   r   torch.nnr   r$   r!   r   Moduler   r'   r(   r   r1   r&   r5   r;   r<   r?   r@   rE   rF   rG   rU   rW   rZ   r`   ra   rb   rc   r
   r
   r
   r   <module>   s2    	
					


