o
    پi                     @   s|  d Z ddlZddlmZ ddlmZ dd Zdd ZG d	d
 d
ejj	Z
d9ddZG dd dejZdd Zdd ZG dd dejj	Zd9ddZG dd dejZd9defddZdd ZG dd  d ejj	Zd9defd!d"ZG d#d$ d$ejZd%d& Zd'd( ZG d)d* d*ejj	Zd9d+d,ZG d-d. d.ejZd/d0 Zd1d2 ZG d3d4 d4ejj	Z d9defd5d6Z!G d7d8 d8ejZ"dS ):a~   Activations (memory-efficient w/ custom autograd)

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

These activations are not compatible with jit scripting or ONNX export of the model, please use
basic versions of the activations.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalc                 C   s   |  t| S N)multorchsigmoidx r
   N/home/ubuntu/.local/lib/python3.10/site-packages/timm/layers/activations_me.py	swish_fwd   s   r   c                 C   s"   t | }||d| d|     S N   )r   r   )r	   grad_output	x_sigmoidr
   r
   r   	swish_bwd   s   
r   c                   @   4   e Zd ZdZedd Zedd Zedd ZdS )	SwishAutoFnz optimised Swish w/ memory-efficient checkpoint
    Inspired by conversation btw Jeremy Howard & Adam Pazske
    https://twitter.com/jeremyphoward/status/1188251041835315200
    c                 C   s   |  d||  d|S )NMulSigmoid)op)gr	   r
   r
   r   symbolic   s   zSwishAutoFn.symbolicc                 C      |  | t|S r   )save_for_backwardr   ctxr	   r
   r
   r   forward#      
zSwishAutoFn.forwardc                 C      | j d }t||S Nr   )saved_tensorsr   r   r   r	   r
   r
   r   backward(      

zSwishAutoFn.backwardN)__name__
__module____qualname____doc__staticmethodr   r   r#   r
   r
   r
   r   r      s    

r   Fc                 C   
   t | S r   r   applyr	   inplacer
   r
   r   swish_me.      
r/   c                       ,   e Zd Zddef fddZdd Z  ZS )SwishMeFr.   c                       t t|   d S r   )superr2   __init__selfr.   	__class__r
   r   r5   3      zSwishMe.__init__c                 C   
   t |S r   r+   r7   r	   r
   r
   r   r   6   r0   zSwishMe.forwardFr%   r&   r'   boolr5   r   __classcell__r
   r
   r8   r   r2   2       r2   c                 C   s   |  tt| S r   )r   r   tanhFsoftplusr   r
   r
   r   mish_fwd:   s   rE   c                 C   s6   t | }t|  }||| | d||    S r   )r   r   rC   rD   rB   r   )r	   r   r   	x_tanh_spr
   r
   r   mish_bwd>   s   
rG   c                   @   (   e Zd ZdZedd Zedd ZdS )
MishAutoFnz Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    A memory efficient variant of Mish
    c                 C   r   r   )r   rE   r   r
   r
   r   r   H   r   zMishAutoFn.forwardc                 C   r   r    )r!   rG   r"   r
   r
   r   r#   M   r$   zMishAutoFn.backwardNr%   r&   r'   r(   r)   r   r#   r
   r
   r
   r   rI   D   s    
rI   c                 C   r*   r   rI   r,   r-   r
   r
   r   mish_meS   r0   rL   c                       r1   )MishMeFr.   c                    r3   r   )r4   rM   r5   r6   r8   r
   r   r5   X   r:   zMishMe.__init__c                 C   r;   r   rK   r<   r
   r
   r   r   [   r0   zMishMe.forwardr=   r>   r
   r
   r8   r   rM   W   rA   rM   r.   c                 C   s   | d j ddddS N   r      minmax      @clampdivr-   r
   r
   r   hard_sigmoid_fwd_   s   rX   c                 C   s&   t | | dk| dk@  d }|| S )N            @rT   )r   	ones_liker	   r   mr
   r
   r   hard_sigmoid_bwdc   s   r^   c                   @   s$   e Zd Zedd Zedd ZdS )HardSigmoidAutoFnc                 C   r   r   )r   rX   r   r
   r
   r   r   i   r   zHardSigmoidAutoFn.forwardc                 C   r   r    )r!   r^   r"   r
   r
   r   r#   n   r$   zHardSigmoidAutoFn.backwardN)r%   r&   r'   r)   r   r#   r
   r
   r
   r   r_   h   s
    
r_   c                 C   r*   r   r_   r,   r-   r
   r
   r   hard_sigmoid_met   r0   ra   c                       r1   )HardSigmoidMeFr.   c                    r3   r   )r4   rb   r5   r6   r8   r
   r   r5   y   r:   zHardSigmoidMe.__init__c                 C   r;   r   r`   r<   r
   r
   r   r   |   r0   zHardSigmoidMe.forwardr=   r>   r
   r
   r8   r   rb   x   rA   rb   c                 C   s   | | d j dddd S rN   rU   r   r
   r
   r   hard_swish_fwd   s   rc   c                 C   s<   t | | dk }t | dk| dk@ | d d |}|| S )NrZ   rY         ?r   r[   wherer\   r
   r
   r   hard_swish_bwd   s   "rg   c                   @   r   )	HardSwishAutoFnz'A memory efficient HardSwish activationc                 C   r   r   )r   rc   r   r
   r
   r   r      r   zHardSwishAutoFn.forwardc                 C   r   r    )r!   rg   r"   r
   r
   r   r#      r$   zHardSwishAutoFn.backwardc                 C   s   |  d|| j dtjdtjdd}|  d|| j dtjdtjdd| j dtjdtjdd}|  d	|| j dtjdtjdd}|  d
||S )NAddConstantrO   )dtype)value_tClipr   rP   Divr   )r   r   tensorfloat)r   r7   input	hardtanh_r
   r
   r   r      s   $<$zHardSwishAutoFn.symbolicN)r%   r&   r'   r(   r)   r   r#   r   r
   r
   r
   r   rh      s    

rh   c                 C   r*   r   rh   r,   r-   r
   r
   r   hard_swish_me   r0   rt   c                       r1   )HardSwishMeFr.   c                    r3   r   )r4   ru   r5   r6   r8   r
   r   r5      r:   zHardSwishMe.__init__c                 C   r;   r   rs   r<   r
   r
   r   r      r0   zHardSwishMe.forwardr=   r>   r
   r
   r8   r   ru      rA   ru   c                 C   s   d|  | d j ddd S )Nrd      r   rQ   )rV   r   r
   r
   r   hard_mish_fwd   s   rw   c                 C   s8   t | | dk }t | dk| dk@ | d |}|| S )Ng       g        g      ?re   r\   r
   r
   r   hard_mish_bwd   s   rx   c                   @   rH   )HardMishAutoFnz A memory efficient variant of Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    c                 C   r   r   )r   rw   r   r
   r
   r   r      r   zHardMishAutoFn.forwardc                 C   r   r    )r!   rx   r"   r
   r
   r   r#      r$   zHardMishAutoFn.backwardNrJ   r
   r
   r
   r   ry      s    
ry   c                 C   r*   r   ry   r,   r-   r
   r
   r   hard_mish_me   r0   r{   c                       r1   )
HardMishMeFr.   c                    r3   r   )r4   r|   r5   r6   r8   r
   r   r5      r:   zHardMishMe.__init__c                 C   r;   r   rz   r<   r
   r
   r   r      r0   zHardMishMe.forwardr=   r>   r
   r
   r8   r   r|      rA   r|   r=   )#r(   r   r   torch.nnr   rC   r   r   autogradFunctionr   r/   Moduler2   rE   rG   rI   rL   rM   r?   rX   r^   r_   ra   rb   rc   rg   rh   rt   ru   rw   rx   ry   r{   r|   r
   r
   r
   r   <module>   s:    


