o
    پi                  
   @   s  d Z ddlmZ ddlmZ ddlmZmZmZm	Z	 ddl
Z
ddlmZ ddlm  mZ ddl
mZ ddlmZ ddlmZmZ dd	lmZmZmZmZmZmZmZmZ d
dlm Z  d
dl!m"Z" d
dl#m$Z$m%Z% d
dl&m'Z'm(Z( dgZ)G dd dej*Z+G dd dej*Z,G dd dej*Z-G dd dej*Z.G dd dej*Z/G dd dej*Z0G dd deZ1G dd deZ2G d d! d!ejZ3G d"d# d#ej*Z4G d$d% d%ej*Z5G d&d' d'ej*Z6G d(d) d)ej*Z7G d*d+ d+ej*Z8G d,d dej*Z9d-d. Z:dd0d1Z;dd3d4Z<e'i d5e<d6d7d8d9e<d6d7d8d:e<d6d7d8d;e<d6d<d8d=e<d6d<d8d>e<d6d?d@e<d6d?dAe<d6d?dBe<d6d?dCe<d6d?dDe<d6dEdFdGe<d6dEdHdIdJdKe<d6dEdFdLe<d6dEdHdIdJdMe<d6dEdNdOdPe<d6dEdFdQe<d6dEdHdIdJi dRe<d6dEdFdSe<d6dEdHdIdJdTe<d6dEdNdOdUe<d6dEdFdVe<d6dEdHdIdJdWe<d6dEdFdXe<d6dEdHdIdJdYe<d6dEdNdOdZe<d6dEdFd[e<d6dEdHdIdJd\e<d6dEdFd]e<d6dEdHdIdJd^e<d6dEdNdOd_e<d6dEdFd`e<d6dEdHdIdJdae<d6dEdFdbe<d6dEdHdIdJi dce<d6dEdNdOdde<d6dEdFdee<d6dEdHdIdJdfe<d6dEdFdge<d6dEdHdIdJdhe<d6dEdNdOdie<d6dEdFdje<d6dEdHdIdJdke<d6dEdFdle<d6dEdHdIdJdme<d6dEdNdOdne<d6dEdFdoe<d6dEdHdIdJdpe<d6dEdFdqe<d6dEdHdIdJdre<d6dEdNdOZ=e(ddse9fdtduZ>e(ddse9fdvdwZ?e(ddse9fdxdyZ@e(ddse9fdzd{ZAe(ddse9fd|d}ZBe(ddse9fd~dZCe(ddse9fddZDe(ddse9fddZEe(ddse9fddZFe(ddse9fddZGe(ddse9fddZHe(ddse9fddZIe(ddse9fddZJe(ddse9fddZKe(ddse9fddZLe(ddse9fddZMe(ddse9fddZNe(ddse9fddZOdS )a  
Poolformer from MetaFormer is Actually What You Need for Vision https://arxiv.org/abs/2111.11418

IdentityFormer, RandFormer, PoolFormerV2, ConvFormer, and CAFormer
from MetaFormer Baselines for Vision https://arxiv.org/abs/2210.13452

All implemented models support feature extraction and variable input resolution.

Original implementation by Weihao Yu et al.,
adapted for timm by Fredo Guan and Ross Wightman.

Adapted from https://github.com/sail-sg/metaformer, original copyright below
    )OrderedDict)partial)ListOptionalTupleUnionN)Tensor)FinalIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)trunc_normal_DropPathSelectAdaptivePool2d
GroupNorm1	LayerNormLayerNorm2dMlpuse_fused_attn   )build_model_with_cfg)feature_take_indices)
checkpointcheckpoint_seq)generate_default_cfgsregister_model
MetaFormerc                       s,   e Zd ZdZ	d fdd	Zdd Z  ZS )Stemzc
    Stem implemented by a layer of convolution.
    Conv2d params constant across all models.
    Nc                    s@   t    tj||dddd| _|r||| _d S t | _d S )N         kernel_sizestridepadding)super__init__nnConv2dconvIdentitynorm)selfin_channelsout_channels
norm_layer	__class__ J/home/ubuntu/.local/lib/python3.10/site-packages/timm/models/metaformer.pyr&   8   s   
 zStem.__init__c                 C      |  |}| |}|S N)r)   r+   r,   xr2   r2   r3   forwardH      

zStem.forwardr5   __name__
__module____qualname____doc__r&   r8   __classcell__r2   r2   r0   r3   r   2   s
    	r   c                       s0   e Zd ZdZ			d	 fdd	Zdd Z  ZS )
Downsamplingz=
    Downsampling implemented by a layer of convolution.
    r   r   Nc                    s:   t    |r||nt | _tj|||||d| _d S )Nr!   )r%   r&   r'   r*   r+   r(   r)   )r,   r-   r.   r"   r#   r$   r/   r0   r2   r3   r&   S   s   
	zDownsampling.__init__c                 C   r4   r5   )r+   r)   r6   r2   r2   r3   r8   f   r9   zDownsampling.forward)r   r   Nr:   r2   r2   r0   r3   r@   N   s    	r@   c                       s*   e Zd ZdZd fdd	Zdd Z  ZS )	Scalez2
    Scale vector by element multiplications.
          ?Tc                    s>   t    |r|ddfn|f| _tj|t| |d| _d S )Nr   requires_grad)r%   r&   shaper'   	Parametertorchonesscale)r,   dim
init_value	trainableuse_nchwr0   r2   r3   r&   q   s   
zScale.__init__c                 C   s   || j | j S r5   )rI   viewrE   r6   r2   r2   r3   r8   v   s   zScale.forward)rB   TTr:   r2   r2   r0   r3   rA   l       rA   c                       *   e Zd ZdZd fdd	Zdd Z  ZS )SquaredReLUz<
        Squared ReLU: https://arxiv.org/abs/2109.08668
    Fc                    s   t    tj|d| _d S )Ninplace)r%   r&   r'   ReLUrelu)r,   rS   r0   r2   r3   r&      s   
zSquaredReLU.__init__c                 C   s   t | |S r5   )rG   squarerU   r6   r2   r2   r3   r8      s   zSquaredReLU.forwardFr:   r2   r2   r0   r3   rQ   z   s    rQ   c                       s6   e Zd ZdZ						d fdd	Zd	d
 Z  ZS )StarReLUz(
    StarReLU: s * relu(x) ** 2 + b
    rB           TNFc                    sV   t    || _tj|d| _tj|td |d| _	tj|td |d| _
d S )NrR   r   rC   )r%   r&   rS   r'   rT   rU   rF   rG   rH   rI   bias)r,   scale_value
bias_valuescale_learnablebias_learnablemoderS   r0   r2   r3   r&      s
   
	zStarReLU.__init__c                 C   s   | j | |d  | j S )Nr    )rI   rU   rZ   r6   r2   r2   r3   r8      s   zStarReLU.forward)rB   rY   TTNFr:   r2   r2   r0   r3   rX      s    rX   c                       sD   e Zd ZU dZee ed< 						d fdd	Zd	d
 Z  Z	S )	Attentionzl
    Vanilla self-attention from Transformer: https://arxiv.org/abs/1706.03762.
    Modified from timm.
    
fused_attn    NFrY   c           	         s   t    || _|d | _t | _|r|n|| | _| jdkr"d| _| j| j | _tj	|| jd |d| _
t|| _tj	| j||d| _t|| _d S )Ng      r   r      rZ   )r%   r&   head_dimrI   r   ra   	num_headsattention_dimr'   LinearqkvDropout	attn_dropproj	proj_drop)	r,   rJ   re   rf   qkv_biasrk   rm   	proj_biaskwargsr0   r2   r3   r&      s   


zAttention.__init__c           
      C   s   |j \}}}| |||d| j| jddddd}|d\}}}| jr7tj	|||| j
r2| jjndd}n||dd	 | j }	|	jd	d
}	| |	}	|	| }|dd|||}| |}| |}|S )Nrc   r    r   r   r   rY   )	dropout_p)rJ   )rE   ri   reshaperf   re   permuteunbindra   Fscaled_dot_product_attentiontrainingrk   p	transposerI   softmaxrl   rm   )
r,   r7   BNCri   qkvattnr2   r2   r3   r8      s    *


zAttention.forward)rb   NFrY   rY   F)
r;   r<   r=   r>   r	   bool__annotations__r&   r8   r?   r2   r2   r0   r3   r`      s   
 r`   c                          e Zd Z fddZ  ZS )GroupNorm1NoBiasc                    ,   t  j|fi | |dd| _d | _d S Nepsư>r%   r&   getr   rZ   r,   num_channelsrp   r0   r2   r3   r&         
zGroupNorm1NoBias.__init__r;   r<   r=   r&   r?   r2   r2   r0   r3   r          r   c                       r   )LayerNorm2dNoBiasc                    r   r   r   r   r0   r2   r3   r&      r   zLayerNorm2dNoBias.__init__r   r2   r2   r0   r3   r      r   r   c                       r   )LayerNormNoBiasc                    r   r   r   r   r0   r2   r3   r&      r   zLayerNormNoBias.__init__r   r2   r2   r0   r3   r      r   r   c                       s8   e Zd ZdZdeejdddf fdd	Zdd	 Z  Z	S )
SepConvz\
    Inverted separable convolution from MobileNetV2: https://arxiv.org/abs/1801.04381.
    r    Fr   rc   c           
         sj   t    t|| }	tj||	d|d| _| | _tj|	|	|||	|d| _| | _tj|	|d|d| _	d S )Nr   )r"   rZ   )r"   r$   groupsrZ   )
r%   r&   intr'   r(   pwconv1act1dwconvact2pwconv2)
r,   rJ   expansion_ratio
act1_layer
act2_layerrZ   r"   r$   rp   mid_channelsr0   r2   r3   r&      s   
zSepConv.__init__c                 C   6   |  |}| |}| |}| |}| |}|S r5   )r   r   r   r   r   r6   r2   r2   r3   r8   
     




zSepConv.forward)
r;   r<   r=   r>   rX   r'   r*   r&   r8   r?   r2   r2   r0   r3   r      s    r   c                       rP   )PoolingzT
    Implementation of pooling for PoolFormer: https://arxiv.org/abs/2111.11418
    rc   c                    s&   t    tj|d|d dd| _d S )Nr   r    F)r#   r$   count_include_pad)r%   r&   r'   	AvgPool2dpool)r,   	pool_sizerp   r0   r2   r3   r&     s   
zPooling.__init__c                 C   s   |  |}|| S r5   )r   )r,   r7   yr2   r2   r3   r8     s   
zPooling.forward)rc   r:   r2   r2   r0   r3   r     rO   r   c                       s6   e Zd ZdZddeeddf fdd	Zdd	 Z  ZS )
MlpHeadz MLP classification head
      r   rY   Tc           	         s\   t    t|| }tj|||d| _| | _||| _tj|||d| _t	|| _
d S )Nrd   )r%   r&   r   r'   rh   fc1actr+   fc2rj   	head_drop)	r,   rJ   num_classes	mlp_ratio	act_layerr/   	drop_raterZ   hidden_featuresr0   r2   r3   r&   &  s   


zMlpHead.__init__c                 C   r   r5   )r   r   r+   r   r   r6   r2   r2   r3   r8   8  r   zMlpHead.forward)	r;   r<   r=   r>   rQ   r   r&   r8   r?   r2   r2   r0   r3   r   "  s    r   c                	       s<   e Zd ZdZeededddddf	 fdd	Zdd	 Z  Z	S )
MetaFormerBlockz1
    Implementation of one MetaFormer block.
    FrY   TNc                    s  t    tt||	|d}tt||
|d}||| _|d||d|| _|dkr-t|nt | _	|	d ur9| nt | _
|
d urE| nt | _||| _t|td| ||||d| _|dkrft|nt | _|	d urr| nt | _|
d ur| | _d S t | _d S )N)rJ   rK   rM   )rJ   rm   rY   r   )r   rZ   dropuse_convr2   )r%   r&   r   rA   norm1token_mixerr   r'   r*   
drop_path1layer_scale1
res_scale1norm2r   r   mlp
drop_path2layer_scale2
res_scale2)r,   rJ   r   mlp_actmlp_biasr/   rm   	drop_pathrM   layer_scale_init_valueres_scale_init_valuerp   ls_layerrs_layerr0   r2   r3   r&   F  s(   



"zMetaFormerBlock.__init__c              
   C   sP   |  || | | | | }| || | | | 	| }|S r5   )
r   r   r   r   r   r   r   r   r   r   r6   r2   r2   r3   r8   k  s   zMetaFormerBlock.forward)
r;   r<   r=   r>   r   rX   r   r&   r8   r?   r2   r2   r0   r3   r   A  s    %r   c                
       sZ   e Zd Zdejedeeddgd ddf
 fdd	Zej	j
ddd	Zd
efddZ  ZS )MetaFormerStager    FrY   Nc                    sz   t    d	_t
t 	_|krt n	t|ddd|d	_	tj
 	
fddt|D  	_d S )NFrc   r    r   )r"   r#   r$   r/   c                    s6   g | ]}t d
 | 	jd 
qS ))
rJ   r   r   r   r/   rm   r   r   r   rM   r2   )r   rM   ).0idp_ratesrp   r   r   r   r/   out_chsrm   r   r,   r   r2   r3   
<listcomp>  s     
z,MetaFormerStage.__init__.<locals>.<listcomp>)r%   r&   grad_checkpointing
issubclassr`   rM   r'   r*   r@   
downsample
Sequentialrangeblocks)r,   in_chsr   depthr   r   r   downsample_normr/   rm   r   r   r   rp   r0   r   r3   r&   }  s   
"	zMetaFormerStage.__init__Tc                 C   s
   || _ d S r5   )r   )r,   enabler2   r2   r3   set_grad_checkpointing  s   
z&MetaFormerStage.set_grad_checkpointingr7   c                 C   s~   |  |}|j\}}}}| js|||ddd}| jr)tj s)t	| j
|}n| 
|}| js=|dd||||}|S )Nrs   r   r    )r   rE   rM   rt   r{   r   rG   jitis_scriptingr   r   )r,   r7   r}   r   HWr2   r2   r3   r8     s   

zMetaFormerStage.forwardT)r;   r<   r=   r'   r*   rX   r   r&   rG   r   ignorer   r   r8   r?   r2   r2   r0   r3   r   {  s    -r   c                       sJ  e Zd ZdZdddddeeddddd	d
eeedf fdd	Zdd Z	e
jjd.ddZe
jjdejfddZd/dedee fddZ						d0de
jdeeeee f  dededededeee
j ee
jee
j f f fd d!Z	"		d1deeee f d#ed$efd%d&Zd2ded'efd(d)Zdefd*d+Zdefd,d-Z  ZS )3r   aM   MetaFormer
        A PyTorch impl of : `MetaFormer Baselines for Vision`  -
          https://arxiv.org/abs/2210.13452

    Args:
        in_chans (int): Number of input image channels.
        num_classes (int): Number of classes for classification head.
        global_pool: Pooling for classifier head.
        depths (list or tuple): Number of blocks at each stage.
        dims (list or tuple): Feature dimension at each stage.
        token_mixers (list, tuple or token_fcn): Token mixer for each stage.
        mlp_act: Activation layer for MLP.
        mlp_bias (boolean): Enable or disable mlp bias term.
        drop_path_rate (float): Stochastic depth rate.
        drop_rate (float): Dropout rate.
        layer_scale_init_values (list, tuple, float or None): Init value for Layer Scale.
            None means not use the layer scale. Form: https://arxiv.org/abs/2103.17239.
        res_scale_init_values (list, tuple, float or None): Init value for res Scale on residual connections.
            None means not use the res scale. From: https://arxiv.org/abs/2110.09456.
        downsample_norm (nn.Module): Norm layer used in stem and downsampling layers.
        norm_layers (list, tuple or norm_fcn): Norm layers for each stage.
        output_norm: Norm layer before classifier head.
        use_mlp_head: Use MLP classification head.
    rc   r   avgr    r       r    @      i@     FrY   N)NNrB   rB   Tc                    sf  t    || _|d | _|| _|| _t|| _t|t	t
fs"|g}t|t	t
fs,|g}t|t	t
fs9|g| j }t|t	t
fsF|g| j }t|t	t
fsS|g| j }t|t	t
fs`|g| j }d| _g | _t||d |d| _g }|d }dd td|	t||D }t| jD ]A}|t||| f|| || |||
|| || || ||| d
|g7 }|| }|  jt|| d|d  d	| d
g7  _qtj| | _|dkr| jrt| j|| jd}| j| _nt| j|}| j| _nt }ttdt|dfd|| jfd|rtdnt fd| jrt |nt fd|fg| _!| "| j# d S )Nrs   Fr   )r/   c                 S   s   g | ]}|  qS r2   )tolist)r   r7   r2   r2   r3   r     s    z'MetaFormer.__init__.<locals>.<listcomp>)
r   r   r   r   rm   r   r   r   r   r/   r    zstages.)num_chs	reductionmoduler   global_pool	pool_typer+   flattenr   r   fc)$r%   r&   r   num_featuresr   use_mlp_headlen
num_stages
isinstancelisttupler   feature_infor   stemrG   linspacesumsplitr   r   dictr'   r   stagesr   head_hidden_sizerh   r*   r   r   Flattenrj   headapply_init_weights)r,   in_chansr   r   depthsdimstoken_mixersr   r   drop_path_rateproj_drop_rater   layer_scale_init_valuesres_scale_init_valuesr   norm_layersoutput_normr   rp   r   prev_dimr   r   finalr0   r2   r3   r&     s   


"
.


zMetaFormer.__init__c                 C   sF   t |tjtjfrt|jdd |jd ur!tj|jd d S d S d S )Ng{Gz?)stdr   )	r   r'   r(   rh   r   weightrZ   init	constant_)r,   mr2   r2   r3   r  ;  s   
zMetaFormer._init_weightsc                 C   s"   || _ | jD ]}|j|d qd S )N)r   )r   r   r   )r,   r   stager2   r2   r3   r   A  s   
z!MetaFormer.set_grad_checkpointingreturnc                 C   s   | j jS r5   )r  r   )r,   r2   r2   r3   get_classifierG  s   zMetaFormer.get_classifierr   r   c                 C   s|   || _ |d urt|d| j_|rtdnt | j_|dkr4| jr,t	| j
|| jd}nt| j
|}nt }|| j_d S )Nr   r   r   r   )r   r   r  r   r'   r   r*   r   r   r   r   r   rh   r   )r,   r   r   r  r2   r2   r3   reset_classifierK  s   zMetaFormer.reset_classifierNCHWr7   indicesr+   
stop_early
output_fmtintermediates_onlyc                 C   s   |dv sJ dg }t t| j|\}}	| |}tj s |s$| j}
n	| jd|	d  }
t|
D ]\}}| jrCtj sCt	||}n||}||v rP|
| q1|rU|S ||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r  zOutput shape must be NCHW.Nr   )r   r   r   r   rG   r   r   	enumerater   r   append)r,   r7   r  r+   r  r  r  intermediatestake_indices	max_indexr   feat_idxr  r2   r2   r3   forward_intermediatesY  s"   

z MetaFormer.forward_intermediatesr   
prune_norm
prune_headc                 C   s<   t t| j|\}}| jd|d  | _|r| dd |S )z@ Prune layers not required for specified intermediates.
        Nr   r    )r   r   r   r  )r,   r  r%  r&  r!  r"  r2   r2   r3   prune_intermediate_layers  s
   z$MetaFormer.prune_intermediate_layers
pre_logitsc                 C   sD   | j |}| j |}| j |}| j |}|r|S | j |S r5   )r  r   r+   r   r   r   )r,   r7   r)  r2   r2   r3   forward_head  s
   zMetaFormer.forward_headc                 C   s8   |  |}| jrtj st| j|}|S | |}|S r5   )r   r   rG   r   r   r   r   r6   r2   r2   r3   forward_features  s   

zMetaFormer.forward_featuresc                 C   r4   r5   )r+  r*  r6   r2   r2   r3   r8     r9   zMetaFormer.forwardr   r5   )NFFr  F)r   FTrW   ) r;   r<   r=   r>   r   rX   r   r   r&   r  rG   r   r   r   r'   Moduler  r   r   strr  r   r   r   r   r   r$  r(  r*  r+  r8   r?   r2   r2   r0   r3   r     sv    a 
/
c                 C   s^  d| v r| S dd l }i }d| v }| }|  D ]\}}|rO|dd|}|dd}|dd	}|d
d}|dd}|dd}|dd}|dd}|dd|}|dd}|dd}|dd|}|dd}|dd}|dd}|dd}|dd |}|d!d"|}|j|| kr| ||  kr||| j}|||< q|S )#Nzstem.conv.weightr   znetwork.0.0.mlp.fc1.weightzlayer_scale_([0-9]+)zlayer_scale\1.scalez	network.1zdownsample_layers.1z	network.3zdownsample_layers.2z	network.5zdownsample_layers.3z	network.2z	network.4z	network.6networkr   zdownsample_layers.([0-9]+)zstages.\1.downsamplezdownsample.projzdownsample.convzpatch_embed.projzpatch_embed.convz([0-9]+).([0-9]+)z\1.blocks.\2zstages.0.downsamplepatch_embedr   	post_normr+   pre_normz^headhead.fcz^normz	head.norm)re
state_dictitemssubreplacerE   numelrt   )r4  modelr3  out_dictis_poolformerv1model_state_dictr   r   r2   r2   r3   checkpoint_filter_fn  s<   "
r=  Fc                 K   sP   t dd t|ddD }|d|}tt| |fttd|dd|}|S )	Nc                 s   s    | ]\}}|V  qd S r5   r2   )r   r   _r2   r2   r3   	<genexpr>  s    z%_create_metaformer.<locals>.<genexpr>r  r   out_indicesT)flatten_sequentialr@  )pretrained_filter_fnfeature_cfg)r   r  r   popr   r   r=  r   )variant
pretrainedrp   default_out_indicesr@  r9  r2   r2   r3   _create_metaformer  s   
	rH  r'  c                 K   s   | dddddt tddd
|S )	Nr   )rc      rI  )r   r   rB   bicubicr2  z	stem.conv)
urlr   
input_sizer   crop_pctinterpolationmeanr  
classifier
first_convr
   )rK  rp   r2   r2   r3   _cfg  s   rR  zpoolformer_s12.sail_in1kztimm/g?)	hf_hub_idrM  zpoolformer_s24.sail_in1kzpoolformer_s36.sail_in1kzpoolformer_m36.sail_in1kgffffff?zpoolformer_m48.sail_in1kzpoolformerv2_s12.sail_in1k)rS  zpoolformerv2_s24.sail_in1kzpoolformerv2_s36.sail_in1kzpoolformerv2_m36.sail_in1kzpoolformerv2_m48.sail_in1kzconvformer_s18.sail_in1kzhead.fc.fc2)rS  rP  zconvformer_s18.sail_in1k_384)rc     rT  )   rU  )rS  rP  rL  r   z!convformer_s18.sail_in22k_ft_in1kz%convformer_s18.sail_in22k_ft_in1k_384zconvformer_s18.sail_in22kiQU  )rS  rP  r   zconvformer_s36.sail_in1kzconvformer_s36.sail_in1k_384z!convformer_s36.sail_in22k_ft_in1kz%convformer_s36.sail_in22k_ft_in1k_384zconvformer_s36.sail_in22kzconvformer_m36.sail_in1kzconvformer_m36.sail_in1k_384z!convformer_m36.sail_in22k_ft_in1kz%convformer_m36.sail_in22k_ft_in1k_384zconvformer_m36.sail_in22kzconvformer_b36.sail_in1kzconvformer_b36.sail_in1k_384z!convformer_b36.sail_in22k_ft_in1kz%convformer_b36.sail_in22k_ft_in1k_384zconvformer_b36.sail_in22kzcaformer_s18.sail_in1kzcaformer_s18.sail_in1k_384zcaformer_s18.sail_in22k_ft_in1kz#caformer_s18.sail_in22k_ft_in1k_384zcaformer_s18.sail_in22kzcaformer_s36.sail_in1kzcaformer_s36.sail_in1k_384zcaformer_s36.sail_in22k_ft_in1kz#caformer_s36.sail_in22k_ft_in1k_384zcaformer_s36.sail_in22kzcaformer_m36.sail_in1kzcaformer_m36.sail_in1k_384zcaformer_m36.sail_in22k_ft_in1kz#caformer_m36.sail_in22k_ft_in1k_384zcaformer_m36.sail_in22kzcaformer_b36.sail_in1kzcaformer_b36.sail_in1k_384zcaformer_b36.sail_in22k_ft_in1kz#caformer_b36.sail_in22k_ft_in1k_384zcaformer_b36.sail_in22kr  c                 K   >   t d	g dg dd tjdtdd dd	|}td
d| i|S )Nr   r   Th㈵>F	r  r  r   r   r   r  r
  r  r   poolformer_s12rF  r2   )rY  r   r'   GELUr   rH  rF  rp   model_kwargsr2   r2   r3   rY       
rY  c                 K   rV  )Nr   r   rU  r   r   TrW  FrX  poolformer_s24rF  r2   )r`  rZ  r\  r2   r2   r3   r`    r^  r`  c                 K   rV  )Nr   r      r   r   Tr   FrX  poolformer_s36rF  r2   )rc  rZ  r\  r2   r2   r3   rc    r^  rc  c                 K   rV  )Nra  `      rT     Tr   FrX  poolformer_m36rF  r2   )rh  rZ  r\  r2   r2   r3   rh    r^  rh  c                 K   rV  )N   rj     rj  rd  Tr   FrX  poolformer_m48rF  r2   )rl  rZ  r\  r2   r2   r3   rl    r^  rl  c                 K   2   t dg dg dtdd|}tdd| i|S )	Nr   r   Fr  r  r  r   poolformerv2_s12rF  r2   )ro  r   r   rH  r\  r2   r2   r3   ro       ro  c                 K   rm  )	Nr_  r   Frn  poolformerv2_s24rF  r2   )rr  rp  r\  r2   r2   r3   rr    rq  rr  c                 K   rm  )	Nra  r   Frn  poolformerv2_s36rF  r2   )rs  rp  r\  r2   r2   r3   rs    rq  rs  c                 K   rm  )	Nra  rd  Frn  poolformerv2_m36rF  r2   )rt  rp  r\  r2   r2   r3   rt    rq  rt  c                 K   rm  )	Nri  rd  Frn  poolformerv2_m48rF  r2   )ru  rp  r\  r2   r2   r3   ru     rq  ru  c                 K   2   t dg dg dttd|}tdd| i|S )Nrc   rc   	   rc   r   r  r  r  r  convformer_s18rF  r2   )rz  r   r   r   rH  r\  r2   r2   r3   rz    rq  rz  c                 K   rv  )Nrc   rU  rb  rc   r   ry  convformer_s36rF  r2   )r}  r{  r\  r2   r2   r3   r}    rq  r}  c                 K   rv  )Nr|  re  rf  rT  i@  ry  convformer_m36rF  r2   )r  r{  r\  r2   r2   r3   r  !  rq  r  c                 K   rv  )Nr|  r      r   rg  ry  convformer_b36rF  r2   )r  r{  r\  r2   r2   r3   r  ,  rq  r  c                 K   J   t dg dg dttttgtgd tgd  d|}tdd| i|S )	Nrw  r   r    ry  caformer_s18rF  r2   )r  r   r   r`   r   r   rH  r\  r2   r2   r3   r  7     
r  c                 K   r  )	Nr|  r   r    ry  caformer_s36rF  r2   )r  r  r\  r2   r2   r3   r  B  r  r  c                 K   r  )	Nr|  r~  r    ry  caformer_m36rF  r2   )r  r  r\  r2   r2   r3   r  M  r  r  c                 K   r  )	Nr|  r  r    ry  caformer_b36rF  r2   )r  r  r\  r2   r2   r3   r  X  r  r  rW   )r'  )Pr>   collectionsr   	functoolsr   typingr   r   r   r   rG   torch.nnr'   torch.nn.functional
functionalrw   r   	torch.jitr	   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   __all__r,  r   r@   rA   rQ   rX   r`   r   r   r   r   r   r   r   r   r   r=  rH  rR  default_cfgsrY  r`  rc  rh  rl  ro  rr  rs  rt  ru  rz  r}  r  r  r  r  r  r  r2   r2   r2   r3   <module>   s   (<#:E l
%






 #'*-037:=@CGJMPSWZ]`cgjmpswz}               











